Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit 3f891a2e by YeYuheng Committed by GitHub
Options

feat: support image dataset search (#6942) 

* feat: support image dataset search

* image recall code

* fix: normalize dataset image inputs

* fix: handle dataset image preview expiry

* update vitest

* fix: track image caption usage metadata

* fix: filter blank embedding queries at callers

* fix: normalize blank dataset search queries

* doc

---------

Co-authored-by: archer <545436317@qq.com>
parent 92e69588
Showing with 3836 additions and 1527 deletions
document/content/self-host/upgrading/4-15/41502.mdx
......@@ -6,6 +6,7 @@ description: 'FastGPT V4.15.0-beta2 更新说明'
## 🚀 新增内容
1. 重写 agentV2 loop 逻辑。
2. 知识库搜索支持原生多模态 embedding 模型以及图搜图。
## ⚙️ 优化
......
document/data/doc-last-modified.json
......@@ -59,8 +59,8 @@
"content/guide/build/workflow/nodes/ai_chat.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/content_extract.en.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/content_extract.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/coreferenceResolution.en.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/coreferenceResolution.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/coreferenceResolution.en.mdx": "2026-05-18T17:26:16+08:00",
"content/guide/build/workflow/nodes/coreferenceResolution.mdx": "2026-05-18T17:26:16+08:00",
"content/guide/build/workflow/nodes/custom_feedback.en.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/custom_feedback.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/dataset_search.en.mdx": "2026-05-07T15:06:40+08:00",
......@@ -69,8 +69,8 @@
"content/guide/build/workflow/nodes/document_parsing.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/form_input.en.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/form_input.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/http.en.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/http.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/http.en.mdx": "2026-05-18T17:26:16+08:00",
"content/guide/build/workflow/nodes/http.mdx": "2026-05-18T17:26:16+08:00",
"content/guide/build/workflow/nodes/knowledge_base_search_merge.en.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/knowledge_base_search_merge.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/laf.en.mdx": "2026-05-07T15:06:40+08:00",
......@@ -85,12 +85,12 @@
"content/guide/build/workflow/nodes/reply.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/sandbox-v2.en.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/sandbox-v2.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/text_editor.en.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/text_editor.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/text_editor.en.mdx": "2026-05-18T17:26:16+08:00",
"content/guide/build/workflow/nodes/text_editor.mdx": "2026-05-18T17:26:16+08:00",
"content/guide/build/workflow/nodes/tfswitch.en.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/tfswitch.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/tool.en.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/tool.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/tool.en.mdx": "2026-05-18T17:26:16+08:00",
"content/guide/build/workflow/nodes/tool.mdx": "2026-05-18T17:26:16+08:00",
"content/guide/build/workflow/nodes/user-selection.en.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/user-selection.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/variable_update.en.mdx": "2026-05-07T15:06:40+08:00",
......@@ -274,7 +274,7 @@
"content/self-host/upgrading/4-14/4149.en.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/4-14/4149.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/4-15/4150.mdx": "2026-05-09T16:13:01+08:00",
"content/self-host/upgrading/4-15/41502.mdx": "2026-05-17T18:56:43+08:00",
"content/self-host/upgrading/4-15/41502.mdx": "2026-05-18T17:26:16+08:00",
"content/self-host/upgrading/outdated/40.en.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/outdated/40.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/outdated/41.en.mdx": "2026-04-26T21:08:47+08:00",
......@@ -415,6 +415,6 @@
"content/self-host/upgrading/outdated/499.mdx": "2026-05-07T15:06:40+08:00",
"content/self-host/upgrading/upgrade-intruction.en.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/upgrade-intruction.mdx": "2026-04-26T21:08:47+08:00",
"content/toc.en.mdx": "2026-05-14T17:58:58+08:00",
"content/toc.mdx": "2026-05-14T17:58:58+08:00"
"content/toc.en.mdx": "2026-05-18T17:26:16+08:00",
"content/toc.mdx": "2026-05-18T17:26:16+08:00"
}
\ No newline at end of file
eslint.config.mjs
......@@ -20,6 +20,7 @@ export default defineConfig([
files: ['**/*.{ts,tsx}'],
rules: {
'@typescript-eslint/no-explicit-any': 'off',
'@typescript-eslint/no-non-null-asserted-optional-chain': 'off',
'@typescript-eslint/consistent-type-imports': [
'error',
{
......
packages/global/core/workflow/constants.ts
......@@ -190,6 +190,7 @@ export enum NodeInputKeyEnum {
datasetSearchUsingExtensionQuery = 'datasetSearchUsingExtensionQuery',
datasetSearchExtensionModel = 'datasetSearchExtensionModel',
datasetSearchExtensionBg = 'datasetSearchExtensionBg',
datasetSearchInput = 'datasetSearchInput',
collectionFilterMatch = 'collectionFilterMatch',
authTmbId = 'authTmbId',
datasetDeepSearch = 'datasetDeepSearch',
......
packages/global/core/workflow/runtime/type.ts
......@@ -220,17 +220,8 @@ export const DispatchNodeResponseSchema = z
limit: z.number().optional().meta({ description: '限制' }),
searchMode: z.enum(DatasetSearchModeEnum).optional().meta({ description: '搜索模式' }),
embeddingWeight: z.number().optional().meta({ description: '嵌入权重' }),
filteredFileCount: z.number().optional().meta({ description: '过滤的非图片文件数量' }),
queryImages: z
.array(
z.object({
key: z.string().optional(),
url: z.string().optional(),
name: z.string().optional()
})
)
.optional()
.meta({ description: '参与知识库检索的图片' }),
datasetQueries: z.array(z.string()).optional().meta({ description: '检索词' }),
rerankModel: z.string().optional().meta({ description: '重排模型' }),
rerankWeight: z.number().optional().meta({ description: '重排权重' }),
reRankInputTokens: z.number().optional().meta({ description: '重排输入 token' }),
......
packages/global/core/workflow/template/system/datasetSearch.ts
......@@ -126,6 +126,14 @@ export const DatasetSearchModule: FlowNodeTemplateType = {
},
{
...Input_Template_UserChatInput,
toolDescription: i18nT('workflow:content_to_search'),
deprecated: true
},
{
...Input_Template_UserChatInput,
label: i18nT('workflow:search_query'),
key: NodeInputKeyEnum.datasetSearchInput,
valueType: WorkflowIOValueTypeEnum.arrayString,
toolDescription: i18nT('workflow:content_to_search')
},
{
......
packages/global/test/core/workflow/template/datasetSearch.test.ts 0 → 100644
import { describe, expect, it } from 'vitest';
import { NodeInputKeyEnum, WorkflowIOValueTypeEnum } from '@fastgpt/global/core/workflow/constants';
import { DatasetSearchModule } from '@fastgpt/global/core/workflow/template/system/datasetSearch';
describe('DatasetSearchModule', () => {
it('should deprecate legacy user question input and add array search input', () => {
const legacyUserQuestionInput = DatasetSearchModule.inputs.find(
(input) => input.key === NodeInputKeyEnum.userChatInput
);
const searchInput = DatasetSearchModule.inputs.find(
(input) => input.key === NodeInputKeyEnum.datasetSearchInput
);
expect(legacyUserQuestionInput?.valueType).toBe(WorkflowIOValueTypeEnum.string);
expect(legacyUserQuestionInput?.deprecated).toBe(true);
expect(searchInput?.valueType).toBe(WorkflowIOValueTypeEnum.arrayString);
});
});
packages/service/common/api/type.d.ts
import type {
DeepRagSearchProps,
SearchDatasetDataResponse
} from '../../core/dataset/search/controller';
import type { DeepRagSearchProps, SearchDatasetDataResponse } from '../../core/dataset/search';
import type { AuthOpenApiLimitProps } from '../../support/openapi/auth';
import type {
CreateUsageProps,
......
packages/service/common/vectorDB/controller.ts
......@@ -3,7 +3,7 @@ import { PgVectorCtrl } from './pg';
import { ObVectorCtrl } from './oceanbase';
import { SeekVectorCtrl } from './seekdb';
import { OpenGaussVectorCtrl } from './opengauss';
import { getVectorsByText } from '../../core/ai/embedding';
import { getVectors } from '../../core/ai/embedding';
import type { VectorControllerType, InsertVectorControllerPropsType } from './type';
import { type EmbeddingModelItemType } from '@fastgpt/global/core/ai/model.schema';
import {
......@@ -111,9 +111,19 @@ export const insertDatasetDataVector = async ({
inputs: string[];
model: EmbeddingModelItemType;
}) => {
const { vectors, tokens } = await getVectorsByText({
if (inputs.length === 0) {
return {
tokens: 0,
insertIds: []
};
}
const { vectors, tokens } = await getVectors({
model,
input: inputs,
inputs: inputs.map((text) => ({
type: 'text',
input: text
})),
type: 'db'
});
const { insertIds } = await retryFn(() =>
......
packages/service/core/ai/embedding/index.ts
......@@ -4,34 +4,67 @@ import { countPromptTokens } from '../../../common/string/tiktoken/index';
import { EmbeddingTypeEnm } from '@fastgpt/global/core/ai/constants';
import { retryFn } from '@fastgpt/global/common/system/utils';
import { getLogger, LogCategories } from '../../../common/logger';
import z from 'zod';
const logger = getLogger(LogCategories.MODULE.AI.EMBEDDING);
type GetVectorProps = {
type GetVectorsBaseProps = {
model: EmbeddingModelItemType;
input: string[] | string;
type?: `${EmbeddingTypeEnm}`;
headers?: Record<string, string>;
};
// text to vector
export async function getVectorsByText({ model, input, type, headers }: GetVectorProps) {
if (!input) {
const InputItemSchema = z.object({
type: z.enum(['text', 'image']),
input: z.string()
});
type GetVectorInputItem = z.infer<typeof InputItemSchema>;
export type GetVectorsProps = GetVectorsBaseProps & {
inputs: GetVectorInputItem[];
};
const getRequestInput = (input: GetVectorInputItem) => {
if (input.type === 'image') {
return {
type: 'image_url',
image_url: {
url: input.input
}
};
}
return input.input;
};
const countInputTokens = async (input: GetVectorInputItem) => {
if (input.type === 'image') return 1;
return countPromptTokens(input.input);
};
export async function getVectors({ model, inputs: rawInputs, type, headers }: GetVectorsProps) {
const inputs = z
.array(InputItemSchema)
.parse(rawInputs)
.map((item) => ({
...item,
input: item.input.trim()
}));
if (inputs.length === 0 || inputs.some((item) => !item.input)) {
return Promise.reject({
code: 500,
message: 'input is empty'
});
}
const { ai } = getAIApi();
const formatInput = Array.isArray(input) ? input : [input];
const { ai } = getAIApi();
let chunkSize = Number(model.batchSize || 1);
chunkSize = isNaN(chunkSize) ? 1 : chunkSize;
const chunks = [];
for (let i = 0; i < formatInput.length; i += chunkSize) {
chunks.push(formatInput.slice(i, i + chunkSize));
for (let i = 0; i < inputs.length; i += chunkSize) {
chunks.push(inputs.slice(i, i + chunkSize));
}
try {
......@@ -40,18 +73,20 @@ export async function getVectorsByText({ model, input, type, headers }: GetVecto
const allVectors: number[][] = [];
for (const chunk of chunks) {
// input text to vector
const requestInput = chunk.map(getRequestInput);
const inputTypes = Array.from(new Set(chunk.map((item) => item.type)));
const result = await retryFn(() =>
ai.embeddings
.create(
{
model: model.model,
input: chunk,
input: requestInput,
encoding_format: 'float',
...model.defaultConfig,
...(type === EmbeddingTypeEnm.db && model.dbConfig),
...(type === EmbeddingTypeEnm.query && model.queryConfig)
},
} as any,
model.requestUrl
? {
path: model.requestUrl,
......@@ -66,17 +101,19 @@ export async function getVectorsByText({ model, input, type, headers }: GetVecto
if (!res.data) {
logger.error('Embedding API returned empty data', {
model: model.model,
inputLength: chunk.length,
inputTypes,
inputCount: chunk.length,
response: res
});
return Promise.reject('Embedding API is not responding');
}
if (!res?.data?.[0]?.embedding) {
// @ts-ignore
// @ts-expect-error provider error payload is not part of the embedding response type
const msg = res.data?.err?.message || '';
logger.error('Embedding API returned invalid embedding', {
model: model.model,
inputLength: chunk.length,
inputTypes,
inputCount: chunk.length,
response: res,
apiMessage: msg
});
......@@ -87,7 +124,7 @@ export async function getVectorsByText({ model, input, type, headers }: GetVecto
(async () => {
if (res.usage) return res.usage.total_tokens;
const tokens = await Promise.all(chunk.map((item) => countPromptTokens(item)));
const tokens = await Promise.all(chunk.map(countInputTokens));
return tokens.reduce((sum, item) => sum + item, 0);
})(),
Promise.all(
......@@ -115,7 +152,8 @@ export async function getVectorsByText({ model, input, type, headers }: GetVecto
} catch (error) {
logger.error('Embedding request failed', {
model: model.model,
inputLengths: formatInput.map((item) => item.length),
inputTypes: Array.from(new Set(inputs.map((item) => item.type))),
inputCount: inputs.length,
error
});
......
packages/service/core/ai/functions/queryExtension.ts
......@@ -250,7 +250,21 @@ export const queryExtension = async ({
const { lazyGreedyQuerySelection, embeddingModel: useEmbeddingModel } = useTextCosine({
embeddingModel
});
queries = queries.map((item) => String(item));
queries = queries.map((item) => String(item).trim()).filter(Boolean);
if (queries.length === 0) {
return {
rawQuery: query,
extensionQueries: [],
llmModel: modelData.model,
embeddingModel,
requestId,
seconds,
inputTokens,
outputTokens,
usedUserOpenAIKey,
embeddingTokens: 0
};
}
const { selectedData: selectedQueries, embeddingTokens } = await lazyGreedyQuerySelection({
originalText: query,
......
packages/service/core/ai/hooks/useTextCosine.ts
......@@ -3,7 +3,7 @@
Reference: https://github.com/jina-ai/submodular-optimization
*/
import { getVectorsByText } from '../embedding';
import { getVectors } from '../embedding';
import { getEmbeddingModel } from '../model';
class PriorityQueue<T> {
......@@ -83,16 +83,28 @@ export const useTextCosine = ({ embeddingModel }: { embeddingModel: string }) =>
k: number;
alpha?: number;
}) => {
const { tokens: embeddingTokens, vectors: embeddingVectors } = await getVectorsByText({
const query = originalText.trim();
const normalizedCandidates = candidates.map((item) => item.trim()).filter(Boolean);
if (!query || normalizedCandidates.length === 0 || k <= 0) {
return {
selectedData: [],
embeddingTokens: 0
};
}
const { tokens: embeddingTokens, vectors: embeddingVectors } = await getVectors({
model: vectorModel,
input: [originalText, ...candidates],
inputs: [query, ...normalizedCandidates].map((text) => ({
type: 'text',
input: text
})),
type: 'query'
});
const originalEmbedding = embeddingVectors[0];
const candidateEmbeddings = embeddingVectors.slice(1);
const n = candidates.length;
const n = normalizedCandidates.length;
const selected: string[] = [];
const selectedEmbeddings: number[][] = [];
......@@ -136,7 +148,7 @@ export const useTextCosine = ({ embeddingModel }: { embeddingModel: string }) =>
}
if (bestCandidate) {
selected.push(candidates[bestCandidate.index]);
selected.push(normalizedCandidates[bestCandidate.index]);
selectedEmbeddings.push(candidateEmbeddings[bestCandidate.index]);
}
}
......
packages/service/core/ai/model.ts
import { cloneDeep } from 'lodash';
import { type SystemModelItemType } from './type';
import type { LLMModelItemType } from '@fastgpt/global/core/ai/model.schema';
import type {
EmbeddingModelItemType,
LLMModelItemType
} from '@fastgpt/global/core/ai/model.schema';
export const getDefaultLLMModel = () => global?.systemDefaultModel.llm!;
export const getDefaultLLMModel = () => global.systemDefaultModel.llm!;
export const getLLMModel = (model?: string | LLMModelItemType) => {
if (!model) return getDefaultLLMModel();
......@@ -30,9 +33,16 @@ export const getDefaultHelperBotModel = (): LLMModelItemType =>
global?.systemDefaultModel.helperBotLLM || getDefaultLLMModel();
export const getDefaultEmbeddingModel = () => global?.systemDefaultModel.embedding!;
export const getEmbeddingModel = (model?: string) => {
export const getEmbeddingModel = (model?: string | EmbeddingModelItemType) => {
if (!model) return getDefaultEmbeddingModel();
return global.embeddingModelMap.get(model) || getDefaultEmbeddingModel();
if (typeof model === 'string') {
return global.embeddingModelMap.get(model) || getDefaultEmbeddingModel();
}
return model;
};
export const isImageEmbeddingModel = (model?: string | EmbeddingModelItemType) => {
return !!getEmbeddingModel(model)?.vision;
};
export const getDefaultTTSModel = () => global?.systemDefaultModel.tts!;
......
packages/service/core/dataset/search/controller.ts deleted 100644 → 0
import {
DatasetSearchModeEnum,
DatasetSearchModeMap,
SearchScoreTypeEnum
} from '@fastgpt/global/core/dataset/constants';
import { recallFromVectorStore } from '../../../common/vectorDB/controller';
import { getVectorsByText } from '../../ai/embedding';
import { getEmbeddingModel, getDefaultRerankModel, getLLMModel } from '../../ai/model';
import { MongoDatasetData } from '../data/schema';
import type {
DatasetCollectionSchemaType,
DatasetDataSchemaType
} from '@fastgpt/global/core/dataset/type';
import {
type DatasetDataTextSchemaType,
type SearchDataResponseItemType
} from '@fastgpt/global/core/dataset/type';
import { MongoDatasetCollection } from '../collection/schema';
import { reRankRecall } from '../../../core/ai/rerank';
import { countPromptTokens } from '../../../common/string/tiktoken/index';
import { datasetSearchResultConcat } from '@fastgpt/global/core/dataset/search/utils';
import { hashStr } from '@fastgpt/global/common/string/tools';
import { jiebaSplit } from '../../../common/string/jieba/index';
import { getCollectionSourceData } from '@fastgpt/global/core/dataset/collection/utils';
import { Types } from '../../../common/mongo';
import json5 from 'json5';
import { MongoDatasetCollectionTags } from '../tag/schema';
import { computeFilterIntersection } from './utils';
import { readFromSecondary } from '../../../common/mongo/utils';
import { MongoDatasetDataText } from '../data/dataTextSchema';
import { type ChatItemMiniType } from '@fastgpt/global/core/chat/type';
import type { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import { datasetSearchQueryExtension } from './utils';
import type { RerankModelItemType } from '@fastgpt/global/core/ai/model.schema';
import { formatDatasetDataValue } from '../data/controller';
import { pushTrack } from '../../../common/middle/tracks/utils';
import { replaceS3KeyToPreviewUrl } from '../../../core/dataset/utils';
import { addDays } from 'date-fns';
import { getLogger, LogCategories } from '../../../common/logger';
import type { OpenaiAccountType } from '@fastgpt/global/support/user/team/type';
const logger = getLogger(LogCategories.MODULE.DATASET.DATA);
export type SearchDatasetDataProps = {
histories: ChatItemMiniType[];
teamId: string;
uid?: string;
tmbId?: string;
model: string;
datasetIds: string[];
reRankQuery: string;
queries: string[];
[NodeInputKeyEnum.datasetSimilarity]?: number; // min distance
[NodeInputKeyEnum.datasetMaxTokens]: number; // max Token limit
[NodeInputKeyEnum.datasetSearchMode]?: DatasetSearchModeEnum;
[NodeInputKeyEnum.datasetSearchEmbeddingWeight]?: number;
[NodeInputKeyEnum.datasetSearchUsingReRank]?: boolean;
[NodeInputKeyEnum.datasetSearchRerankModel]?: RerankModelItemType;
[NodeInputKeyEnum.datasetSearchRerankWeight]?: number;
/*
{
tags: {
$and: ["str1","str2"],
$or: ["str1","str2",null] null means no tags
},
createTime: {
$gte: 'xx',
$lte: 'xxx'
}
}
*/
collectionFilterMatch?: string;
};
export type SearchDatasetDataResponse = {
searchRes: SearchDataResponseItemType[];
embeddingTokens: number;
reRankInputTokens: number;
searchMode: `${DatasetSearchModeEnum}`;
limit: number;
similarity: number;
usingReRank: boolean;
usingSimilarityFilter: boolean;
queryExtensionResult?: {
llmModel: string;
embeddingModel: string;
requestId: string;
seconds: number;
inputTokens: number;
outputTokens: number;
usedUserOpenAIKey: boolean;
embeddingTokens: number;
query: string;
};
deepSearchResult?: { model: string; inputTokens: number; outputTokens: number };
};
export const datasetDataReRank = async ({
rerankModel,
data,
query
}: {
rerankModel?: RerankModelItemType;
data: SearchDataResponseItemType[];
query: string;
}): Promise<{
results: SearchDataResponseItemType[];
inputTokens: number;
}> => {
const { results, inputTokens } = await reRankRecall({
model: rerankModel,
query,
documents: data.map((item) => ({
id: item.id,
text: `${item.q}\n${item.a}`.trim()
}))
});
if (results.length === 0) {
return Promise.reject('Rerank error');
}
// add new score to data
const mergeResult = results
.map((item, index) => {
const target = data.find((dataItem) => dataItem.id === item.id);
if (!target) return null;
const score = item.score || 0;
return {
...target,
score: [{ type: SearchScoreTypeEnum.reRank, value: score, index }]
};
})
.filter(Boolean) as SearchDataResponseItemType[];
return {
results: mergeResult,
inputTokens
};
};
export const filterDatasetDataByMaxTokens = async (
data: SearchDataResponseItemType[],
maxTokens: number
) => {
const filterMaxTokensResult = await (async () => {
// Count tokens
const tokensScoreFilter = await Promise.all(
data.map(async (item) => ({
...item,
tokens: await countPromptTokens(item.q + item.a)
}))
);
const results: SearchDataResponseItemType[] = [];
let totalTokens = 0;
for await (const item of tokensScoreFilter) {
results.push(item);
totalTokens += item.tokens;
if (totalTokens > maxTokens) {
break;
}
}
return results.length === 0 ? data.slice(0, 1) : results;
})();
return filterMaxTokensResult;
};
export async function searchDatasetData(
props: SearchDatasetDataProps
): Promise<SearchDatasetDataResponse> {
const {
teamId,
reRankQuery,
queries,
model,
similarity = 0,
limit: maxTokens,
searchMode: inputSearchMode = DatasetSearchModeEnum.embedding,
embeddingWeight = 0.5,
usingReRank: inputUsingReRank = false,
rerankModel,
rerankWeight = 0.5,
datasetIds = [],
collectionFilterMatch
} = props;
let searchMode = inputSearchMode;
let usingReRank = inputUsingReRank;
// Constants data
const datasetDataSelectField =
'_id datasetId collectionId updateTime q a imageId imageDescMap chunkIndex indexes';
const datsaetCollectionSelectField =
'_id name fileId rawLink apiFileId externalFileId externalFileUrl';
/* init params */
searchMode = DatasetSearchModeMap[searchMode] ? searchMode : DatasetSearchModeEnum.embedding;
usingReRank = usingReRank && !!getDefaultRerankModel();
// Compatible with topk limit
let set = new Set<string>();
let usingSimilarityFilter = false;
/* function */
const countRecallLimit = () => {
if (searchMode === DatasetSearchModeEnum.embedding) {
return {
embeddingLimit: 100,
fullTextLimit: 0
};
}
if (searchMode === DatasetSearchModeEnum.fullTextRecall) {
return {
embeddingLimit: 0,
fullTextLimit: 100
};
}
return {
embeddingLimit: 80,
fullTextLimit: 60
};
};
const getForbidData = async () => {
const collections = await MongoDatasetCollection.find(
{
teamId,
datasetId: { $in: datasetIds },
forbid: true
},
'_id'
);
return {
forbidCollectionIdList: collections.map((item) => String(item._id))
};
};
/*
Collection metadata filter
标签过滤：
1. and 先生效
2. and 标签和 null 不能共存，否则返回空数组
*/
const filterCollectionByMetadata = async (): Promise<string[] | undefined> => {
const getAllCollectionIds = async ({
parentCollectionIds
}: {
parentCollectionIds?: string[];
}): Promise<string[] | undefined> => {
if (!parentCollectionIds) return;
if (parentCollectionIds.length === 0) {
return [];
}
const collections = await MongoDatasetCollection.find(
{
teamId,
datasetId: { $in: datasetIds },
_id: { $in: parentCollectionIds }
},
'_id type',
{
...readFromSecondary
}
).lean();
const resultIds = new Set<string>();
collections.forEach((item) => {
if (item.type !== 'folder') {
resultIds.add(String(item._id));
}
});
const folderIds = collections
.filter((item) => item.type === 'folder')
.map((item) => String(item._id));
// Get all child collection ids
if (folderIds.length) {
const childCollections = await MongoDatasetCollection.find(
{
teamId,
datasetId: { $in: datasetIds },
parentId: { $in: folderIds }
},
'_id type',
{
...readFromSecondary
}
).lean();
const childIds = await getAllCollectionIds({
parentCollectionIds: childCollections.map((item) => String(item._id))
});
childIds?.forEach((id) => resultIds.add(id));
}
return Array.from(resultIds);
};
if (!collectionFilterMatch || !global.feConfigs.isPlus) return;
let tagCollectionIdList: string[] | undefined = undefined;
let createTimeCollectionIdList: string[] | undefined = undefined;
let inputCollectionIdList: string[] | undefined = undefined;
try {
const jsonMatch =
typeof collectionFilterMatch === 'object'
? collectionFilterMatch
: json5.parse(collectionFilterMatch);
const andTags = jsonMatch?.tags?.$and as (string | null)[] | undefined;
const orTags = jsonMatch?.tags?.$or as (string | null)[] | undefined;
if (andTags && andTags.length > 0) {
const uniqueAndTags = Array.from(new Set(andTags));
if (uniqueAndTags.includes(null) && uniqueAndTags.some((tag) => typeof tag === 'string')) {
return [];
}
if (uniqueAndTags.every((tag) => typeof tag === 'string')) {
const matchedTags = await MongoDatasetCollectionTags.find(
{
teamId,
datasetId: { $in: datasetIds },
tag: { $in: uniqueAndTags as string[] }
},
'_id datasetId tag',
{ ...readFromSecondary }
).lean();
// Group tags by dataset
const datasetTagMap = new Map<string, { tagIds: string[]; tagNames: Set<string> }>();
matchedTags.forEach((tag) => {
const datasetId = String(tag.datasetId);
if (!datasetTagMap.has(datasetId)) {
datasetTagMap.set(datasetId, {
tagIds: [],
tagNames: new Set()
});
}
const datasetData = datasetTagMap.get(datasetId)!;
datasetData.tagIds.push(String(tag._id));
datasetData.tagNames.add(tag.tag);
});
const validDatasetIds = Array.from(datasetTagMap.entries())
.filter(([, data]) => uniqueAndTags.every((tag) => data.tagNames.has(tag as string)))
.map(([datasetId]) => datasetId);
if (validDatasetIds.length === 0) return [];
const collectionsPromises = validDatasetIds.map((datasetId) => {
const { tagIds } = datasetTagMap.get(datasetId)!;
return MongoDatasetCollection.find(
{
teamId,
datasetId,
tags: { $all: tagIds }
},
'_id',
{ ...readFromSecondary }
).lean();
});
const collectionsResults = await Promise.all(collectionsPromises);
tagCollectionIdList = collectionsResults.flat().map((item) => String(item._id));
} else if (uniqueAndTags.every((tag) => tag === null)) {
const collections = await MongoDatasetCollection.find(
{
teamId,
datasetId: { $in: datasetIds },
$or: [{ tags: { $size: 0 } }, { tags: { $exists: false } }]
},
'_id',
{ ...readFromSecondary }
).lean();
tagCollectionIdList = collections.map((item) => String(item._id));
}
} else if (orTags && orTags.length > 0) {
// Get tagId by tag string
const orTagArray = await MongoDatasetCollectionTags.find(
{
teamId,
datasetId: { $in: datasetIds },
tag: { $in: orTags.filter((tag) => tag !== null) }
},
'_id',
{ ...readFromSecondary }
).lean();
const orTagIds = orTagArray.map((item) => String(item._id));
// Get collections by tagId
const collections = await MongoDatasetCollection.find(
{
teamId,
datasetId: { $in: datasetIds },
$or: [
{ tags: { $in: orTagIds } },
...(orTags.includes(null) ? [{ tags: { $size: 0 } }] : [])
]
},
'_id',
{ ...readFromSecondary }
).lean();
tagCollectionIdList = collections.map((item) => String(item._id));
}
// time
const getCreateTime = jsonMatch?.createTime?.$gte as string | undefined;
const lteCreateTime = jsonMatch?.createTime?.$lte as string | undefined;
if (getCreateTime || lteCreateTime) {
const collections = await MongoDatasetCollection.find(
{
teamId,
datasetId: { $in: datasetIds },
createTime: {
...(getCreateTime && { $gte: new Date(getCreateTime) }),
...(lteCreateTime && {
$lte: new Date(lteCreateTime)
})
}
},
'_id'
);
createTimeCollectionIdList = collections.map((item) => String(item._id));
}
// collectionIds
const inputCollectionIds = jsonMatch?.collectionIds as string[] | undefined;
if (Array.isArray(inputCollectionIds) && inputCollectionIds.length > 0) {
inputCollectionIdList = await getAllCollectionIds({
parentCollectionIds: inputCollectionIds
});
if (inputCollectionIdList && inputCollectionIdList.length === 0) {
return [];
}
}
// Concat tag, time and collectionIds
const collectionIds = computeFilterIntersection([
tagCollectionIdList,
createTimeCollectionIdList,
inputCollectionIdList
]);
return await getAllCollectionIds({
parentCollectionIds: collectionIds
});
} catch {}
};
const embeddingRecall = async ({
queries,
limit,
forbidCollectionIdList,
filterCollectionIdList
}: {
queries: string[];
limit: number;
forbidCollectionIdList: string[];
filterCollectionIdList?: string[];
}): Promise<{
embeddingRecallResults: SearchDataResponseItemType[][];
tokens: number;
}> => {
if (limit === 0) {
return {
embeddingRecallResults: [],
tokens: 0
};
}
const { vectors, tokens } = await getVectorsByText({
model: getEmbeddingModel(model),
input: queries,
type: 'query'
});
const recallResults = await Promise.all(
vectors.map(async (vector) => {
return await recallFromVectorStore({
teamId,
datasetIds,
vector,
limit,
forbidCollectionIdList,
filterCollectionIdList
});
})
);
// Get data and collections
const collectionIdList = Array.from(
new Set(recallResults.map((item) => item.results.map((item) => item.collectionId)).flat())
);
const indexDataIds = Array.from(
new Set(recallResults.map((item) => item.results.map((item) => item.id?.trim())).flat())
);
const [dataMaps, collectionMaps] = await Promise.all([
MongoDatasetData.find(
{
teamId,
datasetId: { $in: datasetIds },
collectionId: { $in: collectionIdList },
'indexes.dataId': { $in: indexDataIds }
},
datasetDataSelectField,
{ ...readFromSecondary }
)
.lean()
.then((res) => {
const map = new Map<string, DatasetDataSchemaType>();
res.forEach((item) => {
item.indexes.forEach((index) => {
map.set(String(index.dataId), item);
});
});
return map;
}),
MongoDatasetCollection.find(
{
_id: { $in: collectionIdList }
},
datsaetCollectionSelectField,
{ ...readFromSecondary }
)
.lean()
.then((res) => {
const map = new Map<string, DatasetCollectionSchemaType>();
res.forEach((item) => {
map.set(String(item._id), item);
});
return map;
})
]);
const embeddingRecallResults = recallResults.map((item) => {
const set = new Set<string>();
return (
item.results
.map((item, index) => {
const collection = collectionMaps.get(String(item.collectionId));
if (!collection) {
logger.warn('Dataset collection not found during recall', {
collectionId: item.collectionId,
dataId: item.id
});
return;
}
const data = dataMaps.get(String(item.id));
if (!data) {
logger.warn('Dataset data not found during recall', {
dataId: item.id,
collectionId: item.collectionId
});
return;
}
const result: SearchDataResponseItemType = {
id: String(data._id),
updateTime: data.updateTime,
...formatDatasetDataValue({
q: data.q,
a: data.a,
imageId: data.imageId,
imageDescMap: data.imageDescMap
}),
chunkIndex: data.chunkIndex,
datasetId: String(data.datasetId),
collectionId: String(data.collectionId),
...getCollectionSourceData(collection),
score: [{ type: SearchScoreTypeEnum.embedding, value: item?.score || 0, index }]
};
return result;
})
// 多个向量对应一个数据，每一路召回，保障数据只有一份，并且取最高排名
.filter((item) => {
if (!item) return false;
if (set.has(item.id)) return false;
set.add(item.id);
return true;
})
.map((item, index) => {
return {
...item!,
score: item!.score.map((item) => ({ ...item, index }))
};
}) as SearchDataResponseItemType[]
);
});
return {
embeddingRecallResults,
tokens
};
};
const fullTextRecall = async ({
queries,
limit,
filterCollectionIdList,
forbidCollectionIdList
}: {
queries: string[];
limit: number;
filterCollectionIdList?: string[];
forbidCollectionIdList: string[];
}): Promise<{
fullTextRecallResults: SearchDataResponseItemType[][];
}> => {
if (limit === 0) {
return {
fullTextRecallResults: []
};
}
const recallResults = await Promise.all(
queries.map(async (query) => {
return (await MongoDatasetDataText.aggregate(
[
{
$match: {
teamId: new Types.ObjectId(teamId),
$text: { $search: await jiebaSplit({ text: query }) },
datasetId: { $in: datasetIds.map((id) => new Types.ObjectId(id)) },
...(filterCollectionIdList
? {
collectionId: {
$in: filterCollectionIdList
.filter((id) => !forbidCollectionIdList.includes(id))
.map((id) => new Types.ObjectId(id))
}
}
: forbidCollectionIdList?.length
? {
collectionId: {
$nin: forbidCollectionIdList.map((id) => new Types.ObjectId(id))
}
}
: {})
}
},
{
$sort: {
score: { $meta: 'textScore' }
}
},
{
$limit: limit
},
{
$project: {
_id: 1,
collectionId: 1,
dataId: 1,
score: { $meta: 'textScore' }
}
}
],
{
...readFromSecondary
}
)) as (DatasetDataTextSchemaType & { score: number })[];
})
);
const dataIds = Array.from(
new Set(recallResults.map((item) => item.map((item) => item.dataId)).flat())
);
const collectionIds = Array.from(
new Set(recallResults.map((item) => item.map((item) => item.collectionId)).flat())
);
// Get data and collections
const [dataMaps, collectionMaps] = await Promise.all([
MongoDatasetData.find(
{
_id: { $in: dataIds }
},
datasetDataSelectField,
{ ...readFromSecondary }
)
.lean()
.then((res) => {
const map = new Map<string, DatasetDataSchemaType>();
res.forEach((item) => {
map.set(String(item._id), item);
});
return map;
}),
MongoDatasetCollection.find(
{
_id: { $in: collectionIds }
},
datsaetCollectionSelectField,
{ ...readFromSecondary }
)
.lean()
.then((res) => {
const map = new Map<string, DatasetCollectionSchemaType>();
res.forEach((item) => {
map.set(String(item._id), item);
});
return map;
})
]);
const fullTextRecallResults = recallResults.map((item) => {
return item
.map((item, index) => {
const collection = collectionMaps.get(String(item.collectionId));
if (!collection) {
logger.warn('Dataset collection not found during full-text recall', {
collectionId: item.collectionId,
dataId: item.dataId
});
return;
}
const data = dataMaps.get(String(item.dataId));
if (!data) {
logger.warn('Dataset data not found during full-text recall', {
dataId: item.dataId,
collectionId: item.collectionId
});
return;
}
return {
id: String(data._id),
datasetId: String(data.datasetId),
collectionId: String(data.collectionId),
updateTime: data.updateTime,
...formatDatasetDataValue({
q: data.q,
a: data.a,
imageId: data.imageId,
imageDescMap: data.imageDescMap
}),
chunkIndex: data.chunkIndex,
indexes: data.indexes,
...getCollectionSourceData(collection),
score: [
{
type: SearchScoreTypeEnum.fullText,
value: item.score || 0,
index
}
]
};
})
.filter((item) => {
if (!item) return false;
return true;
})
.map((item, index) => {
return {
...item,
score: item!.score.map((item) => ({ ...item, index }))
};
}) as SearchDataResponseItemType[];
});
return {
fullTextRecallResults
};
};
const multiQueryRecall = async ({
embeddingLimit,
fullTextLimit
}: {
embeddingLimit: number;
fullTextLimit: number;
}) => {
const [{ forbidCollectionIdList }, filterCollectionIdList] = await Promise.all([
getForbidData(),
filterCollectionByMetadata()
]);
const [{ tokens, embeddingRecallResults }, { fullTextRecallResults }] = await Promise.all([
embeddingRecall({
queries,
limit: embeddingLimit,
forbidCollectionIdList,
filterCollectionIdList
}),
fullTextRecall({
queries,
limit: fullTextLimit,
filterCollectionIdList,
forbidCollectionIdList
})
]);
// rrf concat
const rrfEmbRecall = datasetSearchResultConcat(
embeddingRecallResults.map((list) => ({ weight: 1, list }))
).slice(0, embeddingLimit);
const rrfFTRecall = datasetSearchResultConcat(
fullTextRecallResults.map((list) => ({ weight: 1, list }))
).slice(0, fullTextLimit);
return {
tokens,
embeddingRecallResults: rrfEmbRecall,
fullTextRecallResults: rrfFTRecall
};
};
/* main step */
// count limit
const { embeddingLimit, fullTextLimit } = countRecallLimit();
// recall
const {
embeddingRecallResults,
fullTextRecallResults,
tokens: embeddingTokens
} = await multiQueryRecall({
embeddingLimit,
fullTextLimit
});
// ReRank results
const { results: reRankResults, inputTokens: reRankInputTokens } = await (async () => {
if (!usingReRank) {
return {
results: [],
inputTokens: 0
};
}
set = new Set<string>(embeddingRecallResults.map((item) => item.id));
const concatRecallResults = embeddingRecallResults.concat(
fullTextRecallResults.filter((item) => !set.has(item.id))
);
// remove same q and a data
set = new Set<string>();
const filterSameDataResults = concatRecallResults.filter((item) => {
// 删除所有的标点符号与空格等，只对文本进行比较
const str = hashStr(`${item.q}${item.a}`.replace(/[^\p{L}\p{N}]/gu, ''));
if (set.has(str)) return false;
set.add(str);
return true;
});
try {
return await datasetDataReRank({
rerankModel,
query: reRankQuery,
data: filterSameDataResults
});
} catch {
usingReRank = false;
return {
results: [],
inputTokens: 0
};
}
})();
const rrfSearchResult = datasetSearchResultConcat([
{ weight: embeddingWeight, list: embeddingRecallResults },
{ weight: 1 - embeddingWeight, list: fullTextRecallResults }
]);
const rrfConcatResults = (() => {
if (reRankResults.length === 0) return rrfSearchResult;
if (rerankWeight === 1) return reRankResults;
return datasetSearchResultConcat([
{ weight: 1 - rerankWeight, list: rrfSearchResult },
{ weight: rerankWeight, list: reRankResults }
]);
})();
// remove same q and a data
set = new Set<string>();
const filterSameDataResults = rrfConcatResults.filter((item) => {
// 删除所有的标点符号与空格等，只对文本进行比较
const str = hashStr(`${item.q}${item.a}`.replace(/[^\p{L}\p{N}]/gu, ''));
if (set.has(str)) return false;
set.add(str);
return true;
});
// score filter
const scoreFilter = (() => {
if (usingReRank) {
usingSimilarityFilter = true;
return filterSameDataResults.filter((item) => {
const reRankScore = item.score.find((item) => item.type === SearchScoreTypeEnum.reRank);
if (reRankScore && reRankScore.value < similarity) return false;
return true;
});
}
if (searchMode === DatasetSearchModeEnum.embedding) {
usingSimilarityFilter = true;
return filterSameDataResults.filter((item) => {
const embeddingScore = item.score.find(
(item) => item.type === SearchScoreTypeEnum.embedding
);
if (embeddingScore && embeddingScore.value < similarity) return false;
return true;
});
}
return filterSameDataResults;
})();
// token filter
const filterMaxTokensResult = await filterDatasetDataByMaxTokens(scoreFilter, maxTokens);
const finalResult = filterMaxTokensResult.map((item) => {
item.q = replaceS3KeyToPreviewUrl(item.q, addDays(new Date(), 90));
return item;
});
pushTrack.datasetSearch({ datasetIds, teamId });
return {
searchRes: finalResult,
embeddingTokens,
reRankInputTokens,
searchMode,
limit: maxTokens,
similarity,
usingReRank,
usingSimilarityFilter
};
}
export type DefaultSearchDatasetDataProps = SearchDatasetDataProps & {
[NodeInputKeyEnum.datasetSearchUsingExtensionQuery]?: boolean;
[NodeInputKeyEnum.datasetSearchExtensionModel]?: string;
[NodeInputKeyEnum.datasetSearchExtensionBg]?: string;
userKey?: OpenaiAccountType;
};
export const defaultSearchDatasetData = async ({
datasetSearchUsingExtensionQuery,
datasetSearchExtensionModel,
datasetSearchExtensionBg,
userKey,
...props
}: DefaultSearchDatasetDataProps): Promise<SearchDatasetDataResponse> => {
const query = props.queries[0];
const histories = props.histories;
const { searchQueries, reRankQuery, aiExtensionResult } = await datasetSearchQueryExtension({
query,
llmModel: datasetSearchUsingExtensionQuery
? getLLMModel(datasetSearchExtensionModel).model
: undefined,
embeddingModel: props.model,
userKey,
extensionBg: datasetSearchExtensionBg,
histories
});
const result = await searchDatasetData({
...props,
reRankQuery: reRankQuery,
queries: searchQueries
});
return {
...result,
queryExtensionResult: aiExtensionResult
? {
llmModel: aiExtensionResult.llmModel,
requestId: aiExtensionResult.requestId,
seconds: aiExtensionResult.seconds,
inputTokens: aiExtensionResult.inputTokens,
outputTokens: aiExtensionResult.outputTokens,
usedUserOpenAIKey: aiExtensionResult.usedUserOpenAIKey,
embeddingModel: aiExtensionResult.embeddingModel,
embeddingTokens: aiExtensionResult.embeddingTokens,
query: searchQueries.join('\n')
}
: undefined
};
};
export type DeepRagSearchProps = SearchDatasetDataProps & {
[NodeInputKeyEnum.datasetDeepSearchModel]?: string;
[NodeInputKeyEnum.datasetDeepSearchMaxTimes]?: number;
[NodeInputKeyEnum.datasetDeepSearchBg]?: string;
};
export const deepRagSearch = (data: DeepRagSearchProps) => global.deepRagHandler(data);
packages/service/core/dataset/search/defaultRecall/collectionFilter.ts 0 → 100644
import json5 from 'json5';
import { MongoDatasetCollection } from '../../collection/schema';
import { MongoDatasetCollectionTags } from '../../tag/schema';
import { readFromSecondary } from '../../../../common/mongo/utils';
import { computeFilterIntersection } from '../utils';
export const getForbidCollectionIdList = async ({
teamId,
datasetIds
}: {
teamId: string;
datasetIds: string[];
}) => {
const collections = await MongoDatasetCollection.find(
{
teamId,
datasetId: { $in: datasetIds },
forbid: true
},
'_id'
);
return collections.map((item) => String(item._id));
};
/**
* 按知识库集合元数据过滤 collectionId。
*
* 标签过滤保持原有语义：`$and` 优先生效，且 `$and` 中字符串标签和 null 不能共存。
* 输入 collectionIds 可以是文件夹，会递归展开为实际文件集合。
*/
export const filterCollectionByMetadata = async ({
teamId,
datasetIds,
collectionFilterMatch
}: {
teamId: string;
datasetIds: string[];
collectionFilterMatch?: string;
}): Promise<string[] | undefined> => {
const getAllCollectionIds = async ({
parentCollectionIds
}: {
parentCollectionIds?: string[];
}): Promise<string[] | undefined> => {
if (!parentCollectionIds) return;
if (parentCollectionIds.length === 0) {
return [];
}
const collections = await MongoDatasetCollection.find(
{
teamId,
datasetId: { $in: datasetIds },
_id: { $in: parentCollectionIds }
},
'_id type',
{
...readFromSecondary
}
).lean();
const resultIds = new Set<string>();
collections.forEach((item) => {
if (item.type !== 'folder') {
resultIds.add(String(item._id));
}
});
const folderIds = collections
.filter((item) => item.type === 'folder')
.map((item) => String(item._id));
// Get all child collection ids
if (folderIds.length) {
const childCollections = await MongoDatasetCollection.find(
{
teamId,
datasetId: { $in: datasetIds },
parentId: { $in: folderIds }
},
'_id type',
{
...readFromSecondary
}
).lean();
const childIds = await getAllCollectionIds({
parentCollectionIds: childCollections.map((item) => String(item._id))
});
childIds?.forEach((id) => resultIds.add(id));
}
return Array.from(resultIds);
};
if (!collectionFilterMatch || !global.feConfigs.isPlus) return;
let tagCollectionIdList: string[] | undefined = undefined;
let createTimeCollectionIdList: string[] | undefined = undefined;
let inputCollectionIdList: string[] | undefined = undefined;
try {
const jsonMatch = json5.parse(collectionFilterMatch);
const andTags = jsonMatch?.tags?.$and as (string | null)[] | undefined;
const orTags = jsonMatch?.tags?.$or as (string | null)[] | undefined;
if (andTags && andTags.length > 0) {
const uniqueAndTags = Array.from(new Set(andTags));
if (uniqueAndTags.includes(null) && uniqueAndTags.some((tag) => typeof tag === 'string')) {
return [];
}
if (uniqueAndTags.every((tag) => typeof tag === 'string')) {
const matchedTags = await MongoDatasetCollectionTags.find(
{
teamId,
datasetId: { $in: datasetIds },
tag: { $in: uniqueAndTags as string[] }
},
'_id datasetId tag',
{ ...readFromSecondary }
).lean();
// Group tags by dataset
const datasetTagMap = new Map<string, { tagIds: string[]; tagNames: Set<string> }>();
matchedTags.forEach((tag) => {
const datasetId = String(tag.datasetId);
if (!datasetTagMap.has(datasetId)) {
datasetTagMap.set(datasetId, {
tagIds: [],
tagNames: new Set()
});
}
const datasetData = datasetTagMap.get(datasetId)!;
datasetData.tagIds.push(String(tag._id));
datasetData.tagNames.add(tag.tag);
});
const validDatasetIds = Array.from(datasetTagMap.entries())
.filter(([, data]) => uniqueAndTags.every((tag) => data.tagNames.has(tag as string)))
.map(([datasetId]) => datasetId);
if (validDatasetIds.length === 0) return [];
const collectionsPromises = validDatasetIds.map((datasetId) => {
const { tagIds } = datasetTagMap.get(datasetId)!;
return MongoDatasetCollection.find(
{
teamId,
datasetId,
tags: { $all: tagIds }
},
'_id',
{ ...readFromSecondary }
).lean();
});
const collectionsResults = await Promise.all(collectionsPromises);
tagCollectionIdList = collectionsResults.flat().map((item) => String(item._id));
} else if (uniqueAndTags.every((tag) => tag === null)) {
const collections = await MongoDatasetCollection.find(
{
teamId,
datasetId: { $in: datasetIds },
$or: [{ tags: { $size: 0 } }, { tags: { $exists: false } }]
},
'_id',
{ ...readFromSecondary }
).lean();
tagCollectionIdList = collections.map((item) => String(item._id));
}
} else if (orTags && orTags.length > 0) {
// Get tagId by tag string
const orTagArray = await MongoDatasetCollectionTags.find(
{
teamId,
datasetId: { $in: datasetIds },
tag: { $in: orTags.filter((tag) => tag !== null) }
},
'_id',
{ ...readFromSecondary }
).lean();
const orTagIds = orTagArray.map((item) => String(item._id));
// Get collections by tagId
const collections = await MongoDatasetCollection.find(
{
teamId,
datasetId: { $in: datasetIds },
$or: [
{ tags: { $in: orTagIds } },
...(orTags.includes(null) ? [{ tags: { $size: 0 } }] : [])
]
},
'_id',
{ ...readFromSecondary }
).lean();
tagCollectionIdList = collections.map((item) => String(item._id));
}
// time
const getCreateTime = jsonMatch?.createTime?.$gte as string | undefined;
const lteCreateTime = jsonMatch?.createTime?.$lte as string | undefined;
if (getCreateTime || lteCreateTime) {
const collections = await MongoDatasetCollection.find(
{
teamId,
datasetId: { $in: datasetIds },
createTime: {
...(getCreateTime && { $gte: new Date(getCreateTime) }),
...(lteCreateTime && {
$lte: new Date(lteCreateTime)
})
}
},
'_id'
);
createTimeCollectionIdList = collections.map((item) => String(item._id));
}
// collectionIds
const inputCollectionIds = jsonMatch?.collectionIds as string[] | undefined;
if (Array.isArray(inputCollectionIds) && inputCollectionIds.length > 0) {
inputCollectionIdList = await getAllCollectionIds({
parentCollectionIds: inputCollectionIds
});
if (inputCollectionIdList && inputCollectionIdList.length === 0) {
return [];
}
}
// Concat tag, time and collectionIds
const collectionIds = computeFilterIntersection([
tagCollectionIdList,
createTimeCollectionIdList,
inputCollectionIdList
]);
return await getAllCollectionIds({
parentCollectionIds: collectionIds
});
} catch {}
};
packages/service/core/dataset/search/defaultRecall/constant.ts 0 → 100644
/**
* 默认召回只读取生成搜索结果所需字段。
* 使用对象投影可以让字段增删更明确，也避免字符串投影在重构时难以 diff。
*/
export const datasetDataSelectField = {
_id: 1,
datasetId: 1,
collectionId: 1,
updateTime: 1,
q: 1,
a: 1,
imageId: 1,
imageDescMap: 1,
chunkIndex: 1,
indexes: 1
};
/**
* collection 只需要来源展示字段。
* 文件内容、权限等重字段不在召回阶段读取，减少搜索路径的 Mongo 负担。
*/
export const datasetCollectionSelectField = {
_id: 1,
name: 1,
fileId: 1,
rawLink: 1,
apiFileId: 1,
externalFileId: 1,
externalFileUrl: 1
};
packages/service/core/dataset/search/defaultRecall/embeddingRecall.ts 0 → 100644
import { SearchScoreTypeEnum } from '@fastgpt/global/core/dataset/constants';
import type {
DatasetCollectionSchemaType,
DatasetDataSchemaType,
SearchDataResponseItemType
} from '@fastgpt/global/core/dataset/type';
import { recallFromVectorStore } from '../../../../common/vectorDB/controller';
import { getVectors } from '../../../ai/embedding';
import { getEmbeddingModel, isImageEmbeddingModel } from '../../../ai/model';
import { MongoDatasetCollection } from '../../collection/schema';
import { MongoDatasetData } from '../../data/schema';
import { getLogger, LogCategories } from '../../../../common/logger';
import { readFromSecondary } from '../../../../common/mongo/utils';
import { normalizeImageToBase64 } from '../utils';
import { datasetCollectionSelectField, datasetDataSelectField } from './constant';
import { buildSearchResultItem, concatRecallLists } from './result';
const logger = getLogger(LogCategories.MODULE.DATASET.DATA);
type EmbeddingRecallSource = 'text' | 'imageCaption' | 'image';
type VectorRecallTask = {
source: EmbeddingRecallSource;
vector: number[];
};
const emptyEmbeddingRecallResult = () => ({
textEmbeddingRecallResults: [] as SearchDataResponseItemType[],
imageCaptionEmbeddingRecallResults: [] as SearchDataResponseItemType[],
imageVectorRecallResults: [] as SearchDataResponseItemType[]
});
/**
* 构造向量召回任务。
* 文本 query 和图片描述 query 都按 text embedding 处理；原始图片 query 只有在当前
* embedding 模型支持图片时才会转 base64 并参与 image embedding。单张图片解析失败
* 会被跳过，避免影响其他文本或图片任务。
*/
const buildVectorRecallTasks = async ({
model,
textQueries,
imageCaptionQueries,
imageQueries
}: {
model: string;
textQueries: string[];
imageCaptionQueries: string[];
imageQueries: string[];
}): Promise<{
tasks: VectorRecallTask[];
tokens: number;
}> => {
const embeddingModel = getEmbeddingModel(model);
const textTasks = [
...textQueries.map((query) => ({ source: 'text' as const, query })),
...imageCaptionQueries.map((query) => ({ source: 'imageCaption' as const, query }))
]
.map((item) => ({
...item,
query: item.query.trim()
}))
.filter((item) => item.query);
const vectorInputs: {
source: EmbeddingRecallSource;
input: Parameters<typeof getVectors>[0]['inputs'][number];
}[] = textTasks.map((item) => ({
source: item.source,
input: {
type: 'text',
input: item.query
}
}));
const validImageQueries = imageQueries.map((url) => url.trim()).filter(Boolean);
if (validImageQueries.length > 0 && isImageEmbeddingModel(embeddingModel)) {
const imageInputs = (
await Promise.all(
validImageQueries.map(async (url, index) => {
try {
return await normalizeImageToBase64(url);
} catch (error) {
// Image search is additive. A stale or unreadable image should not break text recall
// or other valid images in the same request.
logger.warn('Image embedding normalization failed during dataset search', {
imageIndex: index,
error
});
}
})
)
)
.filter((imageUrl): imageUrl is string => typeof imageUrl === 'string' && !!imageUrl.trim())
.map((imageUrl) => ({
source: 'image' as const,
input: {
type: 'image' as const,
input: imageUrl.trim()
}
}));
vectorInputs.push(...imageInputs);
}
if (vectorInputs.length === 0) {
return {
tasks: [],
tokens: 0
};
}
const { tokens, vectors } = await getVectors({
model: embeddingModel,
inputs: vectorInputs.map((item) => item.input),
type: 'query'
});
const tasks = vectors.map((vector, index) => ({
source: vectorInputs[index].source,
vector
}));
return {
tasks,
tokens
};
};
/**
* 执行 embedding 召回并按 query 来源分组返回。
* 向量库返回的是 index dataId，这里需要再回查 data/collection，补齐 q/a、
* 图片字段和来源信息。每个 query 内先按数据块去重，多个 query 之间再交给 RRF 合并。
*/
export const embeddingRecall = async ({
teamId,
datasetIds,
model,
imageQueries,
textQueries,
imageCaptionQueries,
limit,
forbidCollectionIdList,
filterCollectionIdList
}: {
teamId: string;
datasetIds: string[];
model: string;
imageQueries: string[];
textQueries: string[];
imageCaptionQueries: string[];
limit: number;
forbidCollectionIdList: string[];
filterCollectionIdList?: string[];
}): Promise<
ReturnType<typeof emptyEmbeddingRecallResult> & {
tokens: number;
}
> => {
if (limit === 0) {
return {
...emptyEmbeddingRecallResult(),
tokens: 0
};
}
const { tasks, tokens } = await buildVectorRecallTasks({
model,
textQueries,
imageCaptionQueries,
imageQueries
});
if (tasks.length === 0) {
return {
...emptyEmbeddingRecallResult(),
tokens
};
}
const recallResults = await Promise.all(
tasks.map(async ({ vector }) => {
return await recallFromVectorStore({
teamId,
datasetIds,
vector,
limit,
forbidCollectionIdList,
filterCollectionIdList
});
})
);
const collectionIdList = Array.from(
new Set(recallResults.map((item) => item.results.map((item) => item.collectionId)).flat())
);
const indexDataIds = Array.from(
new Set(recallResults.map((item) => item.results.map((item) => item.id?.trim())).flat())
);
const [dataMaps, collectionMaps] = await Promise.all([
MongoDatasetData.find(
{
teamId,
datasetId: { $in: datasetIds },
collectionId: { $in: collectionIdList },
'indexes.dataId': { $in: indexDataIds }
},
datasetDataSelectField,
{ ...readFromSecondary }
)
.lean()
.then((res) => {
const map = new Map<string, DatasetDataSchemaType>();
res.forEach((item) => {
item.indexes.forEach((index) => {
map.set(String(index.dataId), item);
});
});
return map;
}),
MongoDatasetCollection.find(
{
_id: { $in: collectionIdList }
},
datasetCollectionSelectField,
{ ...readFromSecondary }
)
.lean()
.then((res) => {
const map = new Map<string, DatasetCollectionSchemaType>();
res.forEach((item) => {
map.set(String(item._id), item);
});
return map;
})
]);
const groupedRecallLists: Record<EmbeddingRecallSource, SearchDataResponseItemType[][]> = {
text: [],
imageCaption: [],
image: []
};
recallResults.forEach((recallResult, taskIndex) => {
const task = tasks[taskIndex];
const set = new Set<string>();
const list = recallResult.results
.map((item, index) => {
const collection = collectionMaps.get(String(item.collectionId));
if (!collection) {
logger.warn('Dataset collection not found during recall', {
collectionId: item.collectionId,
dataId: item.id
});
return;
}
const data = dataMaps.get(String(item.id?.trim()));
if (!data) {
logger.warn('Dataset data not found during recall', {
dataId: item.id,
collectionId: item.collectionId
});
return;
}
return buildSearchResultItem({
data,
collection,
score: [{ type: SearchScoreTypeEnum.embedding, value: item?.score || 0, index }]
});
})
.filter((item) => {
if (!item) return false;
if (set.has(item.id)) return false;
set.add(item.id);
return true;
})
.map((item, index) => {
return {
...item!,
score: item!.score.map((item) => ({ ...item, index }))
};
}) as SearchDataResponseItemType[];
groupedRecallLists[task.source].push(list);
});
return {
textEmbeddingRecallResults: concatRecallLists(groupedRecallLists.text, limit),
imageCaptionEmbeddingRecallResults: concatRecallLists(groupedRecallLists.imageCaption, limit),
imageVectorRecallResults: concatRecallLists(groupedRecallLists.image, limit),
tokens
};
};
packages/service/core/dataset/search/defaultRecall/fullTextRecall.ts 0 → 100644
import { SearchScoreTypeEnum } from '@fastgpt/global/core/dataset/constants';
import type {
DatasetCollectionSchemaType,
DatasetDataSchemaType,
DatasetDataTextSchemaType,
SearchDataResponseItemType
} from '@fastgpt/global/core/dataset/type';
import { jiebaSplit } from '../../../../common/string/jieba/index';
import { Types } from '../../../../common/mongo';
import { readFromSecondary } from '../../../../common/mongo/utils';
import { getLogger, LogCategories } from '../../../../common/logger';
import { MongoDatasetCollection } from '../../collection/schema';
import { MongoDatasetDataText } from '../../data/dataTextSchema';
import { MongoDatasetData } from '../../data/schema';
import { datasetCollectionSelectField, datasetDataSelectField } from './constant';
import { buildSearchResultItem, concatRecallLists } from './result';
const logger = getLogger(LogCategories.MODULE.DATASET.DATA);
type FullTextRecallSource = 'text' | 'imageCaption';
/**
* 执行 Mongo full-text 召回并按 query 来源分组返回。
* 目前 full-text 只处理文本类 query：用户文本和图片 caption。原始图片不会进入这里，
* 因为 Mongo 文本索引无法直接消费图片向量或图片 URL。
*/
export const fullTextRecall = async ({
teamId,
datasetIds,
queryGroups,
limit,
filterCollectionIdList,
forbidCollectionIdList
}: {
teamId: string;
datasetIds: string[];
queryGroups: {
source: FullTextRecallSource;
queries: string[];
}[];
limit: number;
filterCollectionIdList?: string[];
forbidCollectionIdList: string[];
}): Promise<{
textFullTextRecallResults: SearchDataResponseItemType[];
imageCaptionFullTextRecallResults: SearchDataResponseItemType[];
}> => {
const queryTasks = queryGroups.flatMap((group) =>
group.queries
.map((query) => query.trim())
.filter(Boolean)
.map((query) => ({ source: group.source, query }))
);
if (limit === 0 || queryTasks.length === 0) {
return {
textFullTextRecallResults: [],
imageCaptionFullTextRecallResults: []
};
}
const recallResults = await Promise.all(
queryTasks.map(async ({ query }) => {
return (await MongoDatasetDataText.aggregate(
[
{
$match: {
teamId: new Types.ObjectId(teamId),
$text: { $search: await jiebaSplit({ text: query }) },
datasetId: { $in: datasetIds.map((id) => new Types.ObjectId(id)) },
...(filterCollectionIdList
? {
collectionId: {
$in: filterCollectionIdList
.filter((id) => !forbidCollectionIdList.includes(id))
.map((id) => new Types.ObjectId(id))
}
}
: forbidCollectionIdList?.length
? {
collectionId: {
$nin: forbidCollectionIdList.map((id) => new Types.ObjectId(id))
}
}
: {})
}
},
{
$sort: {
score: { $meta: 'textScore' }
}
},
{
$limit: limit
},
{
$project: {
_id: 1,
collectionId: 1,
dataId: 1,
score: { $meta: 'textScore' }
}
}
],
{
...readFromSecondary
}
)) as (DatasetDataTextSchemaType & { score: number })[];
})
);
const dataIds = Array.from(
new Set(recallResults.map((item) => item.map((item) => item.dataId)).flat())
);
const collectionIds = Array.from(
new Set(recallResults.map((item) => item.map((item) => item.collectionId)).flat())
);
// full-text 表只保存 dataId/collectionId/score，展示字段仍回查主 data 与 collection。
const [dataMaps, collectionMaps] = await Promise.all([
MongoDatasetData.find(
{
_id: { $in: dataIds }
},
datasetDataSelectField,
{ ...readFromSecondary }
)
.lean()
.then((res) => {
const map = new Map<string, DatasetDataSchemaType>();
res.forEach((item) => {
map.set(String(item._id), item);
});
return map;
}),
MongoDatasetCollection.find(
{
_id: { $in: collectionIds }
},
datasetCollectionSelectField,
{ ...readFromSecondary }
)
.lean()
.then((res) => {
const map = new Map<string, DatasetCollectionSchemaType>();
res.forEach((item) => {
map.set(String(item._id), item);
});
return map;
})
]);
const groupedRecallLists: Record<FullTextRecallSource, SearchDataResponseItemType[][]> = {
text: [],
imageCaption: []
};
recallResults.forEach((recallResult, taskIndex) => {
const task = queryTasks[taskIndex];
const list = recallResult
.map((item, index) => {
const collection = collectionMaps.get(String(item.collectionId));
if (!collection) {
logger.warn('Dataset collection not found during full-text recall', {
collectionId: item.collectionId,
dataId: item.dataId
});
return;
}
const data = dataMaps.get(String(item.dataId));
if (!data) {
logger.warn('Dataset data not found during full-text recall', {
dataId: item.dataId,
collectionId: item.collectionId
});
return;
}
return buildSearchResultItem({
data,
collection,
includeIndexes: true,
score: [
{
type: SearchScoreTypeEnum.fullText,
value: item.score || 0,
index
}
]
});
})
.filter((item) => {
if (!item) return false;
return true;
})
.map((item, index) => {
return {
...item,
score: item!.score.map((item) => ({ ...item, index }))
};
}) as SearchDataResponseItemType[];
groupedRecallLists[task.source].push(list);
});
return {
textFullTextRecallResults: concatRecallLists(groupedRecallLists.text, limit),
imageCaptionFullTextRecallResults: concatRecallLists(groupedRecallLists.imageCaption, limit)
};
};
packages/service/core/dataset/search/defaultRecall/imageCaption.ts 0 → 100644
import { getLLMModel } from '../../../ai/model';
import { createLLMResponse } from '../../../ai/llm/request';
import { getLogger, LogCategories } from '../../../../common/logger';
import { normalizeImageToBase64 } from '../utils';
import type { OpenaiAccountType } from '@fastgpt/global/support/user/team/type';
const logger = getLogger(LogCategories.MODULE.DATASET.DATA);
type ImageCaptionQueries = {
model?: string;
queries: string[];
requestIds: string[];
inputTokens: number;
outputTokens: number;
seconds: number;
usedUserOpenAIKey: boolean;
};
const emptyImageCaptionQueries = (): ImageCaptionQueries => ({
queries: [],
requestIds: [],
inputTokens: 0,
outputTokens: 0,
seconds: 0,
usedUserOpenAIKey: false
});
/**
* 将图片 query 转成可参与文本召回的图片描述 query。
* VLM 未配置、模型不支持 vision 或单张图片生成失败时都只降级图片描述召回；
* 原始图片仍可能继续走图片向量召回，所以这里不会抛出错误中断搜索。
*/
export const getImageCaptionQueries = async ({
vlmModel,
imageQueries,
userKey
}: {
vlmModel?: string;
imageQueries: string[];
userKey?: OpenaiAccountType;
}): Promise<ImageCaptionQueries> => {
if (!vlmModel || imageQueries.length === 0) {
return emptyImageCaptionQueries();
}
const vlmModelData = getLLMModel(vlmModel);
if (!vlmModelData?.vision) {
return emptyImageCaptionQueries();
}
const results = await Promise.all(
imageQueries.map(async (url, index) => {
try {
const llmStartTime = Date.now();
const {
answerText,
requestId,
usage: { inputTokens, outputTokens, usedUserOpenAIKey }
} = await createLLMResponse({
userKey,
body: {
model: vlmModelData.model,
temperature: 0.1,
stream: true,
useVision: true,
messages: [
{
role: 'user',
content: [
{
type: 'image_url',
image_url: {
url: await normalizeImageToBase64(url)
}
},
{
type: 'text',
text: '请用一句话描述这张图片的主体、场景、颜色、文字和关键视觉特征。只输出描述，不要解释。'
}
]
}
] as any
}
});
return {
query: answerText.trim(),
requestId,
inputTokens,
outputTokens,
seconds: +((Date.now() - llmStartTime) / 1000).toFixed(2),
usedUserOpenAIKey
};
} catch (error) {
logger.warn('Image caption generation failed during dataset search', {
model: vlmModelData.model,
imageIndex: index,
error
});
return {
query: '',
requestId: '',
inputTokens: 0,
outputTokens: 0,
seconds: 0,
usedUserOpenAIKey: false
};
}
})
);
const validResults = results.filter((item) => item.query);
const billableResults = results.filter((item) => item.inputTokens > 0 || item.outputTokens > 0);
return {
model: vlmModelData.model,
queries: validResults.map((item) => item.query),
requestIds: results.map((item) => item.requestId).filter(Boolean),
inputTokens: results.reduce((sum, item) => sum + item.inputTokens, 0),
outputTokens: results.reduce((sum, item) => sum + item.outputTokens, 0),
seconds: results.reduce((sum, item) => sum + item.seconds, 0),
usedUserOpenAIKey:
billableResults.length > 0 && billableResults.every((item) => item.usedUserOpenAIKey)
};
};
packages/service/core/dataset/search/defaultRecall/index.ts 0 → 100644
import {
DatasetSearchModeEnum,
DatasetSearchModeMap
} from '@fastgpt/global/core/dataset/constants';
import { addDays } from 'date-fns';
import { getDefaultRerankModel } from '../../../ai/model';
import { pushTrack } from '../../../../common/middle/tracks/utils';
import { replaceS3KeyToPreviewUrl } from '../../../../core/dataset/utils';
import type { SearchDatasetDataProps, SearchDatasetDataResponse } from '../type';
import { getImageCaptionQueries } from './imageCaption';
import { multiQueryRecall } from './multiQueryRecall';
import { reRankSearchResults } from './rerank';
import {
concatWeightedRecallLists,
filterSearchResultsByScore,
removeDuplicateSearchResults
} from './result';
import { countRecallLimit, filterDatasetDataByMaxTokens } from './utils';
/**
* 执行默认知识库召回主流程。
*
* 这条链路可以按“补充 query -> 多路召回 -> 分组融合 -> 精排/过滤 -> 输出格式化”理解：
* 1. 输入阶段：`textQueries` 是已经完成 query extension 的文本问题；`imageQueries` 是图片
* URL 或 FastGPT 内部对象 key。图片会先尝试生成 caption，把图片补成一条可搜索的文本 query。
* 2. 召回阶段：文本 query 与图片 caption 走 embedding/full-text 召回；原始图片在 embedding
* 模型支持图片时额外走图片向量召回。任一图片处理失败只丢弃该图片路径，不影响其他 query。
* 3. 融合阶段：先分别合并“用户文本”“图片 caption”“图片向量”三类结果，再按图文混合权重合成
* 最终候选集；这样文本搜索保持主导，图片结果作为视觉补充，纯图片搜索则由图片结果主导。
* 4. 精排与过滤阶段：rerank 只作用于用户文本召回，避免文本 rerank 误伤视觉相似结果；最终统一做
* 去重、相似度阈值过滤和 token 上限裁剪。
* 5. 输出阶段：只在返回前把 chunk 内容里的内部图片 key 替换为预览 URL，避免动态 URL 干扰中间去重
* 和召回融合。
*/
export async function searchDatasetData(
props: SearchDatasetDataProps
): Promise<SearchDatasetDataResponse> {
const {
teamId,
reRankQuery,
textQueries,
imageQueries = [],
userKey,
model,
vlmModel,
similarity = 0,
limit: maxTokens,
searchMode: inputSearchMode = DatasetSearchModeEnum.embedding,
embeddingWeight = 0.5,
usingReRank: inputUsingReRank = false,
rerankModel,
rerankWeight = 0.5,
datasetIds = [],
collectionFilterMatch
} = props;
const searchMode = DatasetSearchModeMap[inputSearchMode]
? inputSearchMode
: DatasetSearchModeEnum.embedding;
const usingReRank = inputUsingReRank && !!reRankQuery && !!getDefaultRerankModel();
// Step 1: 图片先尝试转成文本描述。caption 会作为普通文本 query 参与后续召回，
// 这样即使 embedding 模型不支持图片，也能通过 VLM 描述获得一条文本检索路径。
const imageCaptionQueries = await getImageCaptionQueries({
vlmModel,
imageQueries,
userKey
});
// caption 结果需要回传给工作流计费与响应详情；没有生成出有效描述时不输出该段。
const hasImageCaptionUsage =
imageCaptionQueries.inputTokens > 0 || imageCaptionQueries.outputTokens > 0;
const imageCaptionResult: SearchDatasetDataResponse['imageCaptionResult'] =
imageCaptionQueries.model && (imageCaptionQueries.queries.length > 0 || hasImageCaptionUsage)
? {
model: imageCaptionQueries.model,
inputTokens: imageCaptionQueries.inputTokens,
outputTokens: imageCaptionQueries.outputTokens,
requestIds: imageCaptionQueries.requestIds,
seconds: imageCaptionQueries.seconds,
usedUserOpenAIKey: imageCaptionQueries.usedUserOpenAIKey,
queries: imageCaptionQueries.queries
}
: undefined;
// Step 2: 根据搜索模式决定两条召回链路各自取多少候选。
// embedding/full-text/mixed 三种模式最终都走同一个 multiQueryRecall，便于统一过滤集合范围。
const { embeddingLimit, fullTextLimit } = countRecallLimit(searchMode);
const {
textEmbeddingRecallResults,
imageCaptionEmbeddingRecallResults,
imageVectorRecallResults,
textFullTextRecallResults,
imageCaptionFullTextRecallResults,
tokens: embeddingTokens
} = await multiQueryRecall({
teamId,
datasetIds,
model,
imageQueries,
collectionFilterMatch,
embeddingLimit,
fullTextLimit,
textQueries,
imageCaptionQueries: imageCaptionQueries.queries
});
// Step 3: 先在同一语义来源内融合。
// 用户文本召回由 text embedding 与 text full-text 融合；图片描述同理，
// 原始图片向量召回先独立保留，后面再和 caption 召回合并。
const textRecallResults = concatWeightedRecallLists([
{ weight: embeddingWeight, list: textEmbeddingRecallResults },
{ weight: 1 - embeddingWeight, list: textFullTextRecallResults }
]);
const imageCaptionRecallResults = concatWeightedRecallLists([
{ weight: embeddingWeight, list: imageCaptionEmbeddingRecallResults },
{ weight: 1 - embeddingWeight, list: imageCaptionFullTextRecallResults }
]);
// Step 4: rerank 只处理文本召回。
// 图片向量结果和 caption 结果仍按 RRF 融合，避免用文本 rerank 把视觉相似结果误杀。
const {
results: textRerankRecallResults,
inputTokens: reRankInputTokens,
usingReRank: finalUsingReRank
} = await reRankSearchResults({
usingReRank,
textRecallResults,
rerankModel,
query: reRankQuery,
rerankWeight
});
const hasTextQuery = textQueries.some((item) => item.trim());
// Step 5: 合并图片侧结果。caption 是图片的文本解释，权重低于原始图片向量；
// 但当模型不支持图片向量时，caption 仍能单独提供图片相关召回。
const imageRecallResults = concatWeightedRecallLists([
{
weight: imageCaptionRecallResults.length > 0 ? 0.3 : 0,
list: imageCaptionRecallResults
},
{
weight: imageVectorRecallResults.length > 0 ? 0.7 : 0,
list: imageVectorRecallResults
}
]);
// Step 6: 合并文本侧与图片侧结果。
// 纯图片搜索时图片结果权重为 1；图文混合搜索时，文本问题保持主导，图片作为补充约束。
const rrfConcatResults = concatWeightedRecallLists([
{
weight: textRerankRecallResults.length > 0 ? 1 : 0,
list: textRerankRecallResults
},
{
weight: imageRecallResults.length > 0 ? (hasTextQuery ? 0.7 : 1) : 0,
list: imageRecallResults
}
]);
// Step 7: 最终过滤顺序固定为：同内容去重 -> 相似度阈值 -> token 上限。
// 先去重可以避免同一 chunk 因多路召回重复占用相似度过滤和 token 预算。
const filterSameDataResults = removeDuplicateSearchResults(rrfConcatResults);
const { results: scoreFilter, usingSimilarityFilter } = filterSearchResultsByScore({
data: filterSameDataResults,
usingReRank: finalUsingReRank,
searchMode,
similarity
});
const filterMaxTokensResult = await filterDatasetDataByMaxTokens(scoreFilter, maxTokens);
// Step 8: 返回前把 q 中的内部图片 key 转为可预览 URL。
// 只在最终结果处理，避免中间召回和去重阶段混入带过期时间的动态 URL。
const finalResult = filterMaxTokensResult.map((item) => {
item.q = replaceS3KeyToPreviewUrl(item.q, addDays(new Date(), 90));
return item;
});
pushTrack.datasetSearch({ datasetIds, teamId });
return {
searchRes: finalResult,
embeddingTokens,
reRankInputTokens,
searchMode,
limit: maxTokens,
similarity,
usingReRank: finalUsingReRank,
usingSimilarityFilter,
imageCaptionResult
};
}
packages/service/core/dataset/search/defaultRecall/multiQueryRecall.ts 0 → 100644
import { getForbidCollectionIdList, filterCollectionByMetadata } from './collectionFilter';
import { embeddingRecall } from './embeddingRecall';
import { fullTextRecall } from './fullTextRecall';
/**
* 默认召回的并行调度层。
* 这里先统一计算 forbid collection 与 metadata filter，再把同一份 collection 约束
* 下发给 embedding/full-text 两条召回链路，保证两种召回方式看到的集合范围一致。
*/
export const multiQueryRecall = async ({
teamId,
datasetIds,
model,
imageQueries,
collectionFilterMatch,
embeddingLimit,
fullTextLimit,
textQueries,
imageCaptionQueries
}: {
teamId: string;
datasetIds: string[];
model: string;
imageQueries: string[];
collectionFilterMatch?: string;
embeddingLimit: number;
fullTextLimit: number;
textQueries: string[];
imageCaptionQueries: string[];
}) => {
const [forbidCollectionIdList, filterCollectionIdList] = await Promise.all([
getForbidCollectionIdList({
teamId,
datasetIds
}),
filterCollectionByMetadata({
teamId,
datasetIds,
collectionFilterMatch
})
]);
const [
{
tokens,
textEmbeddingRecallResults,
imageCaptionEmbeddingRecallResults,
imageVectorRecallResults
},
{ textFullTextRecallResults, imageCaptionFullTextRecallResults }
] = await Promise.all([
embeddingRecall({
teamId,
datasetIds,
model,
imageQueries,
textQueries,
imageCaptionQueries,
limit: embeddingLimit,
forbidCollectionIdList,
filterCollectionIdList
}),
fullTextRecall({
teamId,
datasetIds,
queryGroups: [
{ source: 'text', queries: textQueries },
{ source: 'imageCaption', queries: imageCaptionQueries }
],
limit: fullTextLimit,
filterCollectionIdList,
forbidCollectionIdList
})
]);
return {
tokens,
textEmbeddingRecallResults,
imageCaptionEmbeddingRecallResults,
imageVectorRecallResults,
textFullTextRecallResults,
imageCaptionFullTextRecallResults
};
};
packages/service/core/dataset/search/defaultRecall/rerank.ts 0 → 100644
import { SearchScoreTypeEnum } from '@fastgpt/global/core/dataset/constants';
import type { RerankModelItemType } from '@fastgpt/global/core/ai/model.schema';
import type { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
import { reRankRecall } from '../../../../core/ai/rerank';
import { concatWeightedRecallLists, removeDuplicateSearchResults } from './result';
const datasetDataReRank = async ({
rerankModel,
data,
query
}: {
rerankModel?: RerankModelItemType;
data: SearchDataResponseItemType[];
query: string;
}): Promise<{
results: SearchDataResponseItemType[];
inputTokens: number;
}> => {
const { results, inputTokens } = await reRankRecall({
model: rerankModel,
query,
documents: data.map((item) => ({
id: item.id,
text: `${item.q}\n${item.a}`.trim()
}))
});
if (results.length === 0) {
return Promise.reject('Rerank error');
}
// add new score to data
const mergeResult = results
.map((item, index) => {
const target = data.find((dataItem) => dataItem.id === item.id);
if (!target) return null;
const score = item.score || 0;
return {
...target,
score: [{ type: SearchScoreTypeEnum.reRank, value: score, index }]
};
})
.filter(Boolean) as SearchDataResponseItemType[];
return {
results: mergeResult,
inputTokens
};
};
/**
* 只对文本召回结果 rerank。图片召回仍通过 RRF 权重参与最终融合，避免图片向量结果被文本 rerank 误杀。
*/
export const reRankSearchResults = async ({
usingReRank,
textRecallResults,
rerankModel,
query,
rerankWeight
}: {
usingReRank: boolean;
textRecallResults: SearchDataResponseItemType[];
rerankModel?: RerankModelItemType;
query: string;
rerankWeight: number;
}): Promise<{
results: SearchDataResponseItemType[];
inputTokens: number;
usingReRank: boolean;
}> => {
if (!usingReRank || !query || textRecallResults.length === 0) {
return {
results: textRecallResults,
inputTokens: 0,
usingReRank: false
};
}
try {
const { results: reRankResults, inputTokens } = await datasetDataReRank({
rerankModel,
query,
data: removeDuplicateSearchResults(textRecallResults)
});
if (rerankWeight === 1) {
return {
results: reRankResults,
inputTokens,
usingReRank: true
};
}
return {
results: concatWeightedRecallLists([
{ weight: 1 - rerankWeight, list: textRecallResults },
{ weight: rerankWeight, list: reRankResults }
]),
inputTokens,
usingReRank: true
};
} catch {
return {
results: textRecallResults,
inputTokens: 0,
usingReRank: false
};
}
};
packages/service/core/dataset/search/defaultRecall/result.ts 0 → 100644
import { hashStr } from '@fastgpt/global/common/string/tools';
import { DatasetSearchModeEnum, SearchScoreTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { getCollectionSourceData } from '@fastgpt/global/core/dataset/collection/utils';
import { datasetSearchResultConcat } from '@fastgpt/global/core/dataset/search/utils';
import type {
DatasetCollectionSchemaType,
DatasetDataSchemaType,
SearchDataResponseItemType
} from '@fastgpt/global/core/dataset/type';
import { formatDatasetDataValue } from '../../data/controller';
/**
* 把召回命中的 data 与 collection 统一整理成搜索结果。
* embedding/full-text 召回只负责生成各自的 score，展示字段和来源字段在这里保持一致。
*/
export const buildSearchResultItem = ({
data,
collection,
score,
includeIndexes = false
}: {
data: DatasetDataSchemaType;
collection: DatasetCollectionSchemaType;
score: SearchDataResponseItemType['score'];
includeIndexes?: boolean;
}): SearchDataResponseItemType => ({
id: String(data._id),
updateTime: data.updateTime,
...formatDatasetDataValue({
q: data.q,
a: data.a,
imageId: data.imageId,
imageDescMap: data.imageDescMap
}),
chunkIndex: data.chunkIndex,
...(includeIndexes ? { indexes: data.indexes } : {}),
datasetId: String(data.datasetId),
collectionId: String(data.collectionId),
...getCollectionSourceData(collection),
score
});
export const concatRecallLists = (lists: SearchDataResponseItemType[][], limit: number) => {
return datasetSearchResultConcat(lists.map((list) => ({ weight: 1, list }))).slice(0, limit);
};
export const concatWeightedRecallLists = (
lists: { weight: number; list: SearchDataResponseItemType[] }[]
) => {
return datasetSearchResultConcat(lists.filter((item) => item.weight > 0 && item.list.length > 0));
};
/**
* 按 q+a 内容去重。召回链路里同一文本块可能同时被文本、图片描述、图片向量命中，
* 这里用归一化后的内容 hash 保留最前面的排序结果。
*/
export const removeDuplicateSearchResults = (data: SearchDataResponseItemType[]) => {
const set = new Set<string>();
return data.filter((item) => {
// 删除所有的标点符号与空格等，只对文本进行比较
const str = hashStr(`${item.q}${item.a}`.replace(/[^\p{L}\p{N}]/gu, ''));
if (set.has(str)) return false;
set.add(str);
return true;
});
};
export const filterSearchResultsByScore = ({
data,
usingReRank,
searchMode,
similarity
}: {
data: SearchDataResponseItemType[];
usingReRank: boolean;
searchMode: DatasetSearchModeEnum;
similarity: number;
}) => {
const scoreType = usingReRank
? SearchScoreTypeEnum.reRank
: searchMode === DatasetSearchModeEnum.embedding
? SearchScoreTypeEnum.embedding
: undefined;
if (!scoreType) {
return {
results: data,
usingSimilarityFilter: false
};
}
return {
results: data.filter((item) => {
const targetScore = item.score.find((item) => item.type === scoreType);
return !targetScore || targetScore.value >= similarity;
}),
usingSimilarityFilter: true
};
};
packages/service/core/dataset/search/defaultRecall/utils.ts 0 → 100644
import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constants';
import type { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
import { countPromptTokens } from '../../../../common/string/tiktoken/index';
/**
* 根据搜索模式分配每条召回链路的候选数量。
* mixed 模式会让 embedding 与 full-text 都多取一批候选，再交给 RRF/权重融合；
* 单一路径模式则关闭另一条召回，避免无意义查询。
*/
export const countRecallLimit = (searchMode: DatasetSearchModeEnum) => {
if (searchMode === DatasetSearchModeEnum.embedding) {
return {
embeddingLimit: 100,
fullTextLimit: 0
};
}
if (searchMode === DatasetSearchModeEnum.fullTextRecall) {
return {
embeddingLimit: 0,
fullTextLimit: 100
};
}
return {
embeddingLimit: 80,
fullTextLimit: 60
};
};
/**
* 按模型上下文 token 上限截断最终引用列表。
* 至少保留第一条结果：否则高质量但单条超长的命中会被全部过滤，导致用户看到
* “无引用”，这比返回一条可裁剪引用更难排查。
*/
export const filterDatasetDataByMaxTokens = async (
data: SearchDataResponseItemType[],
maxTokens: number
) => {
const tokensScoreFilter = await Promise.all(
data.map(async (item) => ({
...item,
tokens: await countPromptTokens(item.q + item.a)
}))
);
const results: SearchDataResponseItemType[] = [];
let totalTokens = 0;
for await (const item of tokensScoreFilter) {
results.push(item);
totalTokens += item.tokens;
if (totalTokens > maxTokens) {
break;
}
}
return results.length === 0 ? data.slice(0, 1) : results;
};
packages/service/core/dataset/search/index.ts 0 → 100644
import type {
DeepRagSearchProps,
DefaultSearchDatasetDataProps,
SearchDatasetDataResponse
} from './type';
import { getLLMModel } from '../../ai/model';
import { searchDatasetData } from './defaultRecall';
import { datasetSearchQueryExtension } from './utils';
export * from './type';
/**
* 知识库搜索统一入口。
*
* 入口层负责搜索前置编排：文本 query extension、rerank query 生成，以及后续搜索方式分发。
* 当前只接入默认召回实现，调用方不直接依赖具体 recall 目录。
*/
export const defaultSearchDatasetData = async ({
datasetSearchUsingExtensionQuery,
datasetSearchExtensionModel,
datasetSearchExtensionBg,
userKey,
...props
}: DefaultSearchDatasetDataProps): Promise<SearchDatasetDataResponse> => {
const textQueries = props.textQueries.map((query) => query.trim()).filter(Boolean);
const query = textQueries.join('\n');
const { searchQueries, reRankQuery, aiExtensionResult } = query
? await datasetSearchQueryExtension({
query,
llmModel: datasetSearchUsingExtensionQuery
? getLLMModel(datasetSearchExtensionModel)?.model
: undefined,
embeddingModel: props.model,
extensionBg: datasetSearchExtensionBg,
histories: props.histories,
userKey
})
: {
searchQueries: [],
reRankQuery: query,
aiExtensionResult: undefined
};
const result = await searchDatasetData({
...props,
userKey,
reRankQuery,
textQueries: searchQueries
});
return {
...result,
queryExtensionResult: aiExtensionResult
? {
llmModel: aiExtensionResult.llmModel,
requestId: aiExtensionResult.requestId,
seconds: aiExtensionResult.seconds,
inputTokens: aiExtensionResult.inputTokens,
outputTokens: aiExtensionResult.outputTokens,
usedUserOpenAIKey: aiExtensionResult.usedUserOpenAIKey,
embeddingModel: aiExtensionResult.embeddingModel,
embeddingTokens: aiExtensionResult.embeddingTokens,
query: searchQueries.join('\n')
}
: undefined
};
};
/**
* Deep RAG 搜索入口由搜索分发层统一暴露，避免具体 recall 实现持有全局 handler 适配。
*/
export const deepRagSearch = (data: DeepRagSearchProps): Promise<SearchDatasetDataResponse> =>
global.deepRagHandler(data);
packages/service/core/dataset/search/type.ts 0 → 100644
import type { RerankModelItemType } from '@fastgpt/global/core/ai/model.schema';
import type { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constants';
import type { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
import type { ChatItemMiniType } from '@fastgpt/global/core/chat/type';
import type { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import type { OpenaiAccountType } from '@fastgpt/global/support/user/team/type';
export type SearchDatasetDataProps = {
histories: ChatItemMiniType[];
teamId: string;
uid?: string;
tmbId?: string;
model: string;
vlmModel?: string;
datasetIds: string[];
reRankQuery: string;
// 工作流入口归一化后的文本 query。
textQueries: string[];
// 工作流入口归一化后的图片 query。
imageQueries?: string[];
// 外部 OpenAI 账号。默认召回里的辅助 LLM 请求需要沿用它来保持计费一致。
userKey?: OpenaiAccountType;
[NodeInputKeyEnum.datasetSimilarity]?: number; // min distance
[NodeInputKeyEnum.datasetMaxTokens]: number; // max Token limit
[NodeInputKeyEnum.datasetSearchMode]?: DatasetSearchModeEnum;
[NodeInputKeyEnum.datasetSearchEmbeddingWeight]?: number;
[NodeInputKeyEnum.datasetSearchUsingReRank]?: boolean;
[NodeInputKeyEnum.datasetSearchRerankModel]?: RerankModelItemType;
[NodeInputKeyEnum.datasetSearchRerankWeight]?: number;
/*
{
tags: {
$and: ["str1","str2"],
$or: ["str1","str2",null] null means no tags
},
createTime: {
$gte: 'xx',
$lte: 'xxx'
}
}
*/
collectionFilterMatch?: string;
};
export type SearchDatasetDataResponse = {
searchRes: SearchDataResponseItemType[];
embeddingTokens: number;
reRankInputTokens: number;
searchMode: `${DatasetSearchModeEnum}`;
limit: number;
similarity: number;
usingReRank: boolean;
usingSimilarityFilter: boolean;
queryExtensionResult?: {
llmModel: string;
embeddingModel: string;
requestId: string;
seconds: number;
inputTokens: number;
outputTokens: number;
usedUserOpenAIKey: boolean;
embeddingTokens: number;
query: string;
};
deepSearchResult?: { model: string; inputTokens: number; outputTokens: number };
imageCaptionResult?: {
model: string;
inputTokens: number;
outputTokens: number;
requestIds: string[];
seconds: number;
usedUserOpenAIKey: boolean;
queries: string[];
};
};
export type DefaultSearchDatasetDataProps = Omit<SearchDatasetDataProps, 'reRankQuery'> & {
[NodeInputKeyEnum.datasetSearchUsingExtensionQuery]?: boolean;
[NodeInputKeyEnum.datasetSearchExtensionModel]?: string;
[NodeInputKeyEnum.datasetSearchExtensionBg]?: string;
userKey?: OpenaiAccountType;
};
export type DeepRagSearchProps = Omit<SearchDatasetDataProps, 'reRankQuery'> & {
[NodeInputKeyEnum.datasetDeepSearchModel]?: string;
[NodeInputKeyEnum.datasetDeepSearchMaxTimes]?: number;
[NodeInputKeyEnum.datasetDeepSearchBg]?: string;
};
packages/service/core/dataset/search/utils.ts
......@@ -3,9 +3,16 @@ import { type ChatItemMiniType } from '@fastgpt/global/core/chat/type';
import { hashStr } from '@fastgpt/global/common/string/tools';
import { getLogger, LogCategories } from '../../../common/logger';
import type { OpenaiAccountType } from '@fastgpt/global/support/user/team/type';
import { getImageBase64 } from '../../../common/file/image/utils';
import { serviceEnv } from '../../../env';
const logger = getLogger(LogCategories.MODULE.DATASET.DATA);
/**
* 计算多个 collection 过滤条件的交集。
* `undefined` 表示当前过滤维度未启用，应被忽略；空数组表示该维度明确无命中，
* 会参与交集并让最终结果为空。
*/
export const computeFilterIntersection = (lists: (string[] | undefined)[]) => {
const validLists = lists.filter((list): list is string[] => list !== undefined);
......@@ -18,6 +25,33 @@ export const computeFilterIntersection = (lists: (string[] | undefined)[]) => {
});
};
/**
* 按环境开关规范化图片输入。
* data URL 已经是模型可读内容，始终原样返回；普通图片 URL 只有
* serviceEnv.MULTIPLE_DATA_TO_BASE64 为 true 时才转成 base64。
* FastGPT 内部对象 key 的鉴权和临时 URL 生成应在入口层完成，避免通用规范化函数
* 混入业务权限和存储来源判断。
* 这里不吞异常，由上层按图片粒度降级，避免一张坏图中断整次检索。
*/
export const normalizeImageToBase64 = async (imageUrl: string) => {
if (imageUrl.startsWith('data:image/')) {
return imageUrl;
}
if (!serviceEnv.MULTIPLE_DATA_TO_BASE64) {
return imageUrl;
}
const { completeBase64 } = await getImageBase64(imageUrl);
return completeBase64;
};
/**
* 对文本查询做 query extension。
* 调用方会先把多个文本 query 合并成一个字符串传入，这里始终按普通字符串处理，
* 不再兼容旧的“query 已经是扩展结果 JSON”分支。扩展失败时返回原始 query，
* 保证搜索主链路不被 LLM 扩展能力影响。
*/
export const datasetSearchQueryExtension = async ({
query,
llmModel,
......@@ -33,49 +67,32 @@ export const datasetSearchQueryExtension = async ({
extensionBg?: string;
histories?: ChatItemMiniType[];
}) => {
const filterSamQuery = (queries: string[]) => {
/**
* query extension 结果可能与原 query 只有标点或空格差异。
* 去重时忽略标点和空白，但保留原始文本，避免影响后续 embedding 和展示。
*/
const filterSameQuery = (queries: string[]) => {
const set = new Set<string>();
const filterSameQueries = queries.filter((item) => {
// 删除所有的标点符号与空格等，只对文本进行比较
const str = hashStr(item.replace(/[^\p{L}\p{N}]/gu, ''));
if (set.has(str)) return false;
set.add(str);
return true;
});
const filterSameQueries = queries
.map((item) => item.trim())
.filter(Boolean)
.filter((item) => {
// 删除所有的标点符号与空格等，只对文本进行比较
const str = hashStr(item.replace(/[^\p{L}\p{N}]/gu, ''));
if (set.has(str)) return false;
set.add(str);
return true;
});
return filterSameQueries;
};
// 检查传入的 query 是否已经进行过扩展
const {
queries: initQueries,
reRankQuery: initReRankQuery,
alreadyExtension
} = (() => {
/* if query already extension, direct parse */
try {
const jsonParse = JSON.parse(query);
const queries: string[] = Array.isArray(jsonParse) ? filterSamQuery(jsonParse) : [query];
const alreadyExtension = Array.isArray(jsonParse);
return {
queries,
reRankQuery: alreadyExtension ? queries.join('\n') : query,
alreadyExtension
};
} catch {
return {
queries: [query],
reRankQuery: query,
alreadyExtension: false
};
}
})();
let queries = initQueries;
let reRankQuery = initReRankQuery;
let queries = [query];
let reRankQuery = query;
// Use LLM to generate extension queries
const aiExtensionResult = await (async () => {
if (!llmModel || !embeddingModel || alreadyExtension) return;
if (!llmModel || !embeddingModel) return;
try {
const result = await queryExtension({
......@@ -94,7 +111,7 @@ export const datasetSearchQueryExtension = async ({
})();
if (aiExtensionResult) {
queries = queries.concat(aiExtensionResult.extensionQueries);
queries = filterSameQuery(queries.concat(aiExtensionResult.extensionQueries));
reRankQuery = queries.join('\n');
}
......
packages/service/core/workflow/dispatch/ai/agent/sub/dataset/index.ts
......@@ -12,7 +12,7 @@ import { MongoDataset } from '../../../../../../dataset/schema';
import {
defaultSearchDatasetData,
type DefaultSearchDatasetDataProps
} from '../../../../../../dataset/search/controller';
} from '../../../../../../dataset/search';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { getLogger, LogCategories } from '../../../../../../../common/logger';
import type { DispatchSubAppResponse } from '../../type';
......@@ -199,8 +199,7 @@ export const dispatchAgentDatasetSearch = async ({
const searchData: DefaultSearchDatasetDataProps = {
histories: [],
teamId,
reRankQuery: query,
queries: [query],
textQueries: [query],
model: vectorModel.model,
similarity: datasetParams.similarity ?? 0.4,
limit: datasetParams.limit || 5000,
......
packages/service/core/workflow/dispatch/ai/toolcall/hooks/useToolRunner.ts
import type { ChatCompletionMessageToolCall } from '@fastgpt/global/core/ai/llm/type';
import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants';
import { chats2GPTMessages } from '@fastgpt/global/core/chat/adapt';
import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
import type { RuntimeNodeItemType } from '@fastgpt/global/core/workflow/runtime/type';
import type { WorkflowInteractiveResponseType } from '@fastgpt/global/core/workflow/template/system/interactive/type';
import type { AgentLoopChildrenInteractiveParams } from '../../../../../ai/llm/agentLoop';
import { runSandboxTools } from '../../../../../ai/sandbox/toolCall';
......@@ -43,11 +46,42 @@ const getAssistantMessages = (assistantResponses: DispatchFlowResponse['assistan
reserveId: false
});
/**
* ToolCall 调用知识库搜索节点时，模型只生成文本 query；父节点输入文件需要在这里追加。
* 这里只做最小拼接，不判断文件类型：搜索节点会统一调用 normalizeDatasetSearchInput
* 拆分文本和图片。清空 userChatInput 是为了让搜索节点走 datasetSearchInput 数组，
* 避免旧字段优先级导致追加的文件 URL 被忽略。
*/
const mergeDatasetToolFileUrls = ({
flowNodeType,
startParams,
fileUrls = []
}: {
flowNodeType: RuntimeNodeItemType['flowNodeType'];
startParams: Record<string, any>;
fileUrls?: string[];
}) => {
if (flowNodeType !== FlowNodeTypeEnum.datasetSearchNode || fileUrls.length === 0) {
return startParams;
}
const queryInput =
startParams[NodeInputKeyEnum.datasetSearchInput] ?? startParams[NodeInputKeyEnum.userChatInput];
const queryList = Array.isArray(queryInput) ? queryInput : queryInput ? [queryInput] : [];
return {
...startParams,
[NodeInputKeyEnum.userChatInput]: '',
[NodeInputKeyEnum.datasetSearchInput]: [...queryList, ...fileUrls]
};
};
export const useToolRunner = ({
workflowProps,
runtimeNodes,
runtimeEdges,
allFiles,
fileUrls = [],
getToolInfo,
cacheToolFlowResponse,
appendToolFlowResponse,
......@@ -57,6 +91,7 @@ export const useToolRunner = ({
runtimeNodes: DispatchToolModuleProps['runtimeNodes'];
runtimeEdges: DispatchToolModuleProps['runtimeEdges'];
allFiles: Map<string, FileInputType>;
fileUrls?: string[];
getToolInfo: (name: string) => ToolInfo | undefined;
cacheToolFlowResponse: (args: {
call: ChatCompletionMessageToolCall;
......@@ -133,7 +168,11 @@ export const useToolRunner = ({
* 用户配置的工具节点会在当前 runtime 副本中被标记为入口节点；
* 参数只注入入口节点，后续依旧走原 workflow 的边和节点调度。
*/
const startParams = parseJsonArgs(call.function.arguments);
const startParams = mergeDatasetToolFileUrls({
flowNodeType: toolNode.flowNodeType,
startParams: parseJsonArgs(call.function.arguments) ?? {},
fileUrls
});
initToolNodes(runtimeNodes, [toolNode.nodeId], startParams);
initToolCallEdges(runtimeEdges, [toolNode.nodeId]);
......
packages/service/core/workflow/dispatch/ai/toolcall/index.ts
import { NodeInputKeyEnum, NodeOutputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import { NodeOutputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants';
import type { DispatchNodeResultType } from '@fastgpt/global/core/workflow/runtime/type';
import { getLLMModel } from '../../../../ai/model';
......@@ -18,7 +18,7 @@ type Response = DispatchNodeResultType<{
export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<Response> => {
const {
node: { nodeId, isEntry, inputs },
node: { nodeId, isEntry },
runtimeNodes,
runtimeEdges,
histories,
......@@ -33,14 +33,13 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
systemPrompt,
userChatInput,
history = 6,
fileUrlList,
fileUrlList: fileLinks,
aiChatVision,
aiChatReasoning,
isResponseAnswerText = true,
useAgentSandbox
}
} = props;
let fileLinks = fileUrlList;
const useSandbox = !!useAgentSandbox && !!global.feConfigs?.show_agent_sandbox;
......@@ -51,10 +50,6 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
props.params.aiChatVision = aiChatVision && toolModel.vision;
props.params.aiChatReasoning = aiChatReasoning && toolModel.reasoning;
const fileUrlInput = inputs.find((item) => item.key === NodeInputKeyEnum.fileUrlList);
if (!fileUrlInput || !fileUrlInput.value || fileUrlInput.value.length === 0) {
fileLinks = undefined;
}
const toolNodes = useToolNodeList({
nodeId,
......
packages/service/core/workflow/dispatch/ai/toolcall/toolCall.ts
......@@ -36,7 +36,6 @@ export const runToolCall = async (props: DispatchToolModuleProps): Promise<Respo
childrenInteractiveParams,
allFiles,
currentInputFiles,
...workflowProps
} = props;
const {
......@@ -60,7 +59,8 @@ export const runToolCall = async (props: DispatchToolModuleProps): Promise<Respo
aiChatReasoning,
aiChatReasoningEffort,
isResponseAnswerText = true,
useAgentSandbox
useAgentSandbox,
fileUrlList
}
} = workflowProps;
......@@ -98,6 +98,7 @@ export const runToolCall = async (props: DispatchToolModuleProps): Promise<Respo
runtimeNodes,
runtimeEdges,
allFiles,
fileUrls: fileUrlList,
getToolInfo,
cacheToolFlowResponse,
appendToolFlowResponse,
......
packages/service/core/workflow/dispatch/ai/toolcall/type.ts
......@@ -35,6 +35,7 @@ export type DispatchToolModuleProps = ModuleDispatchProps<{
childrenInteractiveParams?: AgentLoopChildrenInteractiveParams<WorkflowInteractiveResponseType>;
allFiles: Map<string, FileInputType>;
currentInputFiles: FileInputType[];
fileUrls?: string[];
};
export type ToolNodeItemType = {
......
packages/service/core/workflow/dispatch/dataset/nodeResponse.ts
......@@ -8,17 +8,20 @@ const datasetSearchModuleLogo = 'core/workflow/template/datasetSearch';
const createDatasetSearchChildNodeResponse = ({
requestId,
requestIds,
usage,
moduleName,
seconds,
textOutput
}: {
requestId?: string;
requestIds?: string[];
usage: ChatNodeUsageType;
moduleName: string;
seconds?: number;
textOutput?: string;
}): ChatHistoryItemResType => {
const llmRequestIds = requestIds?.length ? requestIds : requestId ? [requestId] : undefined;
const id = requestId || getNanoid();
return {
......@@ -29,7 +32,7 @@ const createDatasetSearchChildNodeResponse = ({
moduleLogo: datasetSearchModuleLogo,
runningTime: seconds,
model: usage.model,
llmRequestIds: requestId ? [requestId] : undefined,
llmRequestIds,
inputTokens: usage.inputTokens,
outputTokens: usage.outputTokens,
totalPoints: usage.totalPoints,
......@@ -61,6 +64,29 @@ export const createQueryExtensionChildNodeResponse = ({
});
/**
* 创建知识库搜索里的图片解析子 nodeResponse。
* 一次图搜可能解析多张图片，所以这里保留所有 caption LLM requestId。
*/
export const createImageCaptionChildNodeResponse = ({
requestIds,
usage,
seconds,
queries
}: {
requestIds?: string[];
usage: ChatNodeUsageType;
seconds?: number;
queries: string[];
}) =>
createDatasetSearchChildNodeResponse({
requestIds,
usage,
seconds,
moduleName: i18nT('account_usage:image_parse'),
textOutput: queries.join('\n')
});
/**
* 创建知识库搜索里的 chunk selection 子 nodeResponse。
* Agent 知识库工具会在搜索结果过长时触发该 LLM 裁切请求，requestId 只挂在这个子节点上。
*/
......
packages/service/core/workflow/dispatch/dataset/search.ts
......@@ -4,7 +4,7 @@ import type { SelectedDatasetType } from '@fastgpt/global/core/workflow/type/io'
import type { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
import type { ModuleDispatchProps } from '@fastgpt/global/core/workflow/runtime/type';
import { getEmbeddingModel, getRerankModel } from '../../../ai/model';
import { deepRagSearch, defaultSearchDatasetData } from '../../../dataset/search/controller';
import { deepRagSearch, defaultSearchDatasetData } from '../../../dataset/search';
import type { NodeInputKeyEnum, NodeOutputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants';
import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constants';
......@@ -16,7 +16,11 @@ import { getDatasetSearchToolResponsePrompt } from '@fastgpt/global/core/ai/prom
import { getNodeErrResponse } from '../utils';
import { getLogger, LogCategories } from '../../../../common/logger';
import type { ChatHistoryItemResType } from '@fastgpt/global/core/chat/type';
import { createQueryExtensionChildNodeResponse } from './nodeResponse';
import {
createImageCaptionChildNodeResponse,
createQueryExtensionChildNodeResponse
} from './nodeResponse';
import { normalizeDatasetSearchInput } from './utils';
const logger = getLogger(LogCategories.MODULE.WORKFLOW.DATASET);
......@@ -25,6 +29,7 @@ type DatasetSearchProps = ModuleDispatchProps<{
[NodeInputKeyEnum.datasetSimilarity]: number;
[NodeInputKeyEnum.datasetMaxTokens]: number;
[NodeInputKeyEnum.userChatInput]?: string;
[NodeInputKeyEnum.datasetSearchInput]?: string[];
[NodeInputKeyEnum.datasetSearchMode]: DatasetSearchModeEnum;
[NodeInputKeyEnum.datasetSearchEmbeddingWeight]?: number;
......@@ -62,6 +67,7 @@ export async function dispatchDatasetSearch(
similarity,
limit = 5000,
userChatInput = '',
datasetSearchInput = [],
authTmbId = false,
collectionFilterMatch,
searchMode,
......@@ -102,7 +108,10 @@ export async function dispatchDatasetSearch(
[DispatchNodeResponseKeyEnum.toolResponses]: []
};
if (!userChatInput) {
const searchQueries = userChatInput ? [userChatInput] : datasetSearchInput;
const { textQueries, imageQueries } = normalizeDatasetSearchInput(searchQueries);
if (textQueries.length === 0 && imageQueries.length === 0) {
return emptyResult;
}
......@@ -119,9 +128,11 @@ export async function dispatchDatasetSearch(
}
// Get vector model
const vectorModel = getEmbeddingModel(
(await MongoDataset.findById(datasets[0].datasetId, 'vectorModel').lean())?.vectorModel
);
const dataset = await MongoDataset.findById(
datasets[0].datasetId,
'vectorModel vlmModel'
).lean();
const vectorModel = getEmbeddingModel(dataset?.vectorModel);
// Get Rerank Model
const rerankModelData = getRerankModel(rerankModel);
......@@ -129,9 +140,10 @@ export async function dispatchDatasetSearch(
const searchData = {
histories,
teamId,
reRankQuery: userChatInput,
queries: [userChatInput],
textQueries,
imageQueries,
model: vectorModel.model,
vlmModel: dataset?.vlmModel,
similarity,
limit,
datasetIds,
......@@ -142,6 +154,7 @@ export async function dispatchDatasetSearch(
rerankWeight,
collectionFilterMatch
};
const useDeepSearch = datasetDeepSearch && textQueries.length > 0;
const {
searchRes,
embeddingTokens,
......@@ -149,8 +162,9 @@ export async function dispatchDatasetSearch(
usingSimilarityFilter,
usingReRank: searchUsingReRank,
queryExtensionResult,
imageCaptionResult,
deepSearchResult
} = datasetDeepSearch
} = useDeepSearch
? await deepRagSearch({
...searchData,
datasetDeepSearchModel,
......@@ -233,7 +247,32 @@ export async function dispatchDatasetSearch(
outputTokens: 0
});
}
// 4. Deep search
// 4. Image caption
if (imageCaptionResult) {
const { totalPoints, modelName } = formatModelChars2Points({
model: imageCaptionResult.model,
inputTokens: imageCaptionResult.inputTokens,
outputTokens: imageCaptionResult.outputTokens
});
const imageCaptionPoints = imageCaptionResult.usedUserOpenAIKey ? 0 : totalPoints;
const imageCaptionUsage: ChatNodeUsageType = {
totalPoints: imageCaptionPoints,
moduleName: i18nT('account_usage:image_parse'),
model: modelName,
inputTokens: imageCaptionResult.inputTokens,
outputTokens: imageCaptionResult.outputTokens
};
nodeUsages.push(imageCaptionUsage);
childrenResponses.push(
createImageCaptionChildNodeResponse({
requestIds: imageCaptionResult.requestIds,
usage: imageCaptionUsage,
seconds: imageCaptionResult.seconds,
queries: imageCaptionResult.queries
})
);
}
// 5. Deep search
if (deepSearchResult) {
const { totalPoints, modelName } = formatModelChars2Points({
model: deepSearchResult.model,
......@@ -262,7 +301,7 @@ export async function dispatchDatasetSearch(
},
[DispatchNodeResponseKeyEnum.nodeResponse]: {
totalPoints,
query: userChatInput,
datasetQueries: [...textQueries, ...imageQueries],
embeddingModel: vectorModel.name,
embeddingTokens,
similarity: usingSimilarityFilter ? similarity : undefined,
......
packages/service/core/workflow/dispatch/dataset/utils.ts 0 → 100644
import { ChatFileTypeEnum } from '@fastgpt/global/core/chat/constants';
import { parseUrlToFileType } from '../../utils/context';
export type NormalizeDatasetSearchInputResult = {
textQueries: string[];
imageQueries: string[];
};
const httpUrlReg = /^https?:\/\//i;
const pushUnique = <T>(list: T[], seen: Set<T>, value: T) => {
if (!seen.has(value)) {
seen.add(value);
list.push(value);
}
};
const isHttpUrl = (input: string) => httpUrlReg.test(input);
/**
* 将数据集搜索输入拆成文本查询和图片查询。
* datasetSearchInput 会同时接收用户问题和 userFiles；这里只把普通 http(s) URL
* 作为文件候选继续判断，非 http(s) 输入都保留为文本检索 query。
*/
export const normalizeDatasetSearchInput = (
inputList: string[]
): NormalizeDatasetSearchInputResult => {
const textQueries: string[] = [];
const imageQueries: string[] = [];
const seenTextQueries = new Set<string>();
const seenQueryImageUrls = new Set<string>();
for (const rawInput of inputList) {
const input = rawInput.trim();
if (!input) continue;
if (!isHttpUrl(input)) {
pushUnique(textQueries, seenTextQueries, input);
continue;
}
const fileInfo = parseUrlToFileType(input);
if (fileInfo?.type !== ChatFileTypeEnum.image || seenQueryImageUrls.has(input)) {
continue;
}
seenQueryImageUrls.add(input);
imageQueries.push(input);
}
return {
textQueries,
imageQueries
};
};
packages/service/core/workflow/utils/context.ts
......@@ -102,7 +102,7 @@ export const parseUrlToFileType = (url: string): UserChatItemFileItemType | unde
name: filename ? decodeURIComponent(filename) : url,
url
};
} catch (error) {
} catch {
return {
type: ChatFileTypeEnum.file,
name: url,
......
packages/service/test/common/vectorDB/controller.test.ts
......@@ -5,11 +5,10 @@ import {
mockVectorEmbRecall,
mockVectorInit,
mockGetVectorDataByTime,
mockGetVectorCountByTeamId,
mockGetVectorCount,
resetVectorMocks
} from '@test/mocks/common/vector';
import { mockGetVectorsByText } from '@test/mocks/core/ai/embedding';
import { mockGetVectors } from '@test/mocks/core/ai/embedding';
// Import controller functions after mocks are set up
import {
......@@ -65,7 +64,7 @@ describe('VectorDB Controller', () => {
mockSetRedisCache.mockReset();
mockDelRedisCache.mockReset();
mockLoggerWarn.mockReset();
mockGetVectorsByText.mockClear();
mockGetVectors.mockClear();
});
afterEach(() => {
......@@ -248,7 +247,7 @@ describe('VectorDB Controller', () => {
[0.1, 0.2],
[0.3, 0.4]
];
mockGetVectorsByText.mockResolvedValue({
mockGetVectors.mockResolvedValue({
tokens: 100,
vectors: mockVectors
});
......@@ -264,9 +263,18 @@ describe('VectorDB Controller', () => {
model: mockModel as any
});
expect(mockGetVectorsByText).toHaveBeenCalledWith({
expect(mockGetVectors).toHaveBeenCalledWith({
model: mockModel,
input: ['hello world', 'test text'],
inputs: [
{
type: 'text',
input: 'hello world'
},
{
type: 'text',
input: 'test text'
}
],
type: 'db'
});
expect(mockVectorInsert).toHaveBeenCalledWith({
......@@ -282,7 +290,7 @@ describe('VectorDB Controller', () => {
});
it('should invalidate team vector cache after insert', async () => {
mockGetVectorsByText.mockResolvedValue({
mockGetVectors.mockResolvedValue({
tokens: 50,
vectors: [[0.1]]
});
......@@ -302,7 +310,7 @@ describe('VectorDB Controller', () => {
});
it('should return insert result when team vector cache invalidation fails', async () => {
mockGetVectorsByText.mockResolvedValue({
mockGetVectors.mockResolvedValue({
tokens: 50,
vectors: [[0.1]]
});
......@@ -332,7 +340,7 @@ describe('VectorDB Controller', () => {
it('should return insert result when team vector cache invalidation times out', async () => {
vi.useFakeTimers();
mockGetVectorsByText.mockResolvedValue({
mockGetVectors.mockResolvedValue({
tokens: 50,
vectors: [[0.1]]
});
......@@ -363,14 +371,6 @@ describe('VectorDB Controller', () => {
});
it('should handle empty inputs', async () => {
mockGetVectorsByText.mockResolvedValue({
tokens: 0,
vectors: []
});
mockVectorInsert.mockResolvedValue({
insertIds: []
});
const result = await insertDatasetDataVector({
teamId: 'team_123',
datasetId: 'dataset_456',
......@@ -383,6 +383,9 @@ describe('VectorDB Controller', () => {
tokens: 0,
insertIds: []
});
expect(mockGetVectors).not.toHaveBeenCalled();
expect(mockVectorInsert).not.toHaveBeenCalled();
expect(mockDelRedisCache).not.toHaveBeenCalled();
});
});
......
packages/service/test/core/ai/embedding/index.test.ts
......@@ -356,16 +356,16 @@ describe('formatVectors function test', () => {
});
});
describe('getVectorsByText function test', () => {
// The global mock in test/mocks/core/ai/embedding.ts replaces getVectorsByText.
describe('getVectors function test', () => {
// The global mock in test/mocks/core/ai/embedding.ts replaces getVectors.
// Bypass it by importing the actual module implementation.
let getVectorsByText: (typeof import('@fastgpt/service/core/ai/embedding/index'))['getVectorsByText'];
let getVectors: (typeof import('@fastgpt/service/core/ai/embedding/index'))['getVectors'];
beforeAll(async () => {
const actual = await vi.importActual<typeof import('@fastgpt/service/core/ai/embedding/index')>(
'@fastgpt/service/core/ai/embedding/index'
);
getVectorsByText = actual.getVectorsByText;
getVectors = actual.getVectors;
});
beforeEach(() => {
......@@ -390,22 +390,25 @@ describe('getVectorsByText function test', () => {
data: embeddings.map((embedding) => ({ embedding })),
usage: opts.usage
});
const textInput = (input: string) => ({ type: 'text' as const, input });
const imageInput = (input: string) => ({ type: 'image' as const, input });
describe('input validation', () => {
it('should reject with "input is empty" when input is an empty string', async () => {
await expect(getVectorsByText({ model: buildModel(), input: '' })).rejects.toMatchObject({
await expect(
getVectors({ model: buildModel(), inputs: [textInput('')] })
).rejects.toMatchObject({
code: 500,
message: 'input is empty'
});
expect(mockCreate).not.toHaveBeenCalled();
});
it('should reject when input is an empty array (falsy via [].length check indirectly)', async () => {
// Empty array is truthy, so it passes the `!input` guard and proceeds.
// With zero chunks, no API call is made and we get 0 tokens / 0 vectors.
mockCreate.mockResolvedValue(makeResponse([[0.1, 0.2, 0.3, 0.4]]));
const result = await getVectorsByText({ model: buildModel(), input: [] });
expect(result).toEqual({ tokens: 0, vectors: [] });
it('should reject when inputs is empty', async () => {
await expect(getVectors({ model: buildModel(), inputs: [] })).rejects.toMatchObject({
code: 500,
message: 'input is empty'
});
expect(mockCreate).not.toHaveBeenCalled();
});
});
......@@ -416,7 +419,7 @@ describe('getVectorsByText function test', () => {
makeResponse([[0.1, 0.2, 0.3, 0.4]], { usage: { total_tokens: 7 } })
);
const result = await getVectorsByText({ model: buildModel(), input: 'hello' });
const result = await getVectors({ model: buildModel(), inputs: [textInput('hello')] });
expect(mockCreate).toHaveBeenCalledTimes(1);
expect(mockCreate).toHaveBeenCalledWith(
......@@ -458,9 +461,9 @@ describe('getVectorsByText function test', () => {
makeResponse([[2.1, 2.2, 2.3, 2.4]], { usage: { total_tokens: 2 } })
);
const result = await getVectorsByText({
const result = await getVectors({
model: buildModel({ batchSize: 2 }),
input: ['a', 'b', 'c', 'd', 'e']
inputs: ['a', 'b', 'c', 'd', 'e'].map(textInput)
});
expect(mockCreate).toHaveBeenCalledTimes(3);
......@@ -476,9 +479,9 @@ describe('getVectorsByText function test', () => {
makeResponse([[0.1, 0.2, 0.3, 0.4]], { usage: { total_tokens: 1 } })
);
// Pass undefined to exercise `Number(undefined) → NaN` branch
const result = await getVectorsByText({
const result = await getVectors({
model: buildModel({ batchSize: undefined }),
input: ['x', 'y']
inputs: ['x', 'y'].map(textInput)
});
expect(mockCreate).toHaveBeenCalledTimes(2);
......@@ -493,10 +496,74 @@ describe('getVectorsByText function test', () => {
);
mockCreate.mockResolvedValue(makeResponse([base64], { usage: { total_tokens: 2 } }));
const result = await getVectorsByText({ model: buildModel(), input: 'hi' });
const result = await getVectors({ model: buildModel(), inputs: [textInput('hi')] });
expect(result.vectors[0].slice(0, 4)).toEqual(raw);
});
it('should build image_url input parts for image embeddings', async () => {
mockCreate.mockResolvedValue(
makeResponse(
[
[0.1, 0.2, 0.3, 0.4],
[0.5, 0.6, 0.7, 0.8]
],
{ usage: { total_tokens: 6 } }
)
);
const result = await getVectors({
model: buildModel(),
inputs: [imageInput('data:image/png;base64,aaa'), imageInput('data:image/png;base64,bbb')]
});
expect(mockCreate).toHaveBeenCalledTimes(1);
expect(mockCreate.mock.calls[0][0].input).toEqual([
{
type: 'image_url',
image_url: {
url: 'data:image/png;base64,aaa'
}
},
{
type: 'image_url',
image_url: {
url: 'data:image/png;base64,bbb'
}
}
]);
expect(result.tokens).toBe(6);
expect(result.vectors).toHaveLength(2);
});
it('should build mixed text and image input parts in order', async () => {
mockCreate.mockResolvedValue(
makeResponse(
[
[0.1, 0.2, 0.3, 0.4],
[0.5, 0.6, 0.7, 0.8]
],
{ usage: { total_tokens: 8 } }
)
);
const result = await getVectors({
model: buildModel(),
inputs: [textInput('hello'), imageInput('data:image/png;base64,aaa')]
});
expect(mockCreate.mock.calls[0][0].input).toEqual([
'hello',
{
type: 'image_url',
image_url: {
url: 'data:image/png;base64,aaa'
}
}
]);
expect(result.tokens).toBe(8);
expect(result.vectors).toHaveLength(2);
});
});
describe('token counting fallback', () => {
......@@ -504,13 +571,28 @@ describe('getVectorsByText function test', () => {
// No usage → the function computes tokens from chunk strings.
mockCreate.mockResolvedValue(makeResponse([[0.1, 0.2, 0.3, 0.4]]));
const result = await getVectorsByText({ model: buildModel(), input: 'hello world' });
const result = await getVectors({
model: buildModel(),
inputs: [textInput('hello world')]
});
expect(result.vectors).toHaveLength(1);
// countPromptTokens should produce a non-negative integer for a real string
expect(typeof result.tokens).toBe('number');
expect(result.tokens).toBeGreaterThanOrEqual(0);
});
it('should fall back to image count when image embedding response lacks usage', async () => {
mockCreate.mockResolvedValue(makeResponse([[0.1, 0.2, 0.3, 0.4]]));
const result = await getVectors({
model: buildModel(),
inputs: [imageInput('data:image/png;base64,aaa')]
});
expect(result.tokens).toBe(1);
expect(result.vectors).toHaveLength(1);
});
});
describe('type / config selection', () => {
......@@ -520,7 +602,7 @@ describe('getVectorsByText function test', () => {
);
const model = buildModel({ defaultConfig: { dimensions: 512 } as any });
await getVectorsByText({ model, input: 'x' });
await getVectors({ model, inputs: [textInput('x')] });
expect(mockCreate.mock.calls[0][0]).toMatchObject({ dimensions: 512 });
});
......@@ -531,7 +613,7 @@ describe('getVectorsByText function test', () => {
);
const model = buildModel({ dbConfig: { input_type: 'passage' } as any });
await getVectorsByText({ model, input: 'x', type: EmbeddingTypeEnm.db });
await getVectors({ model, inputs: [textInput('x')], type: EmbeddingTypeEnm.db });
expect(mockCreate.mock.calls[0][0]).toMatchObject({ input_type: 'passage' });
});
......@@ -542,7 +624,7 @@ describe('getVectorsByText function test', () => {
);
const model = buildModel({ queryConfig: { input_type: 'query' } as any });
await getVectorsByText({ model, input: 'x', type: EmbeddingTypeEnm.query });
await getVectors({ model, inputs: [textInput('x')], type: EmbeddingTypeEnm.query });
expect(mockCreate.mock.calls[0][0]).toMatchObject({ input_type: 'query' });
});
......@@ -556,7 +638,7 @@ describe('getVectorsByText function test', () => {
queryConfig: { input_type: 'query' } as any
});
await getVectorsByText({ model, input: 'x' });
await getVectors({ model, inputs: [textInput('x')] });
expect(mockCreate.mock.calls[0][0]).not.toHaveProperty('input_type');
});
......@@ -572,9 +654,9 @@ describe('getVectorsByText function test', () => {
requestAuth: 'secret-token'
});
await getVectorsByText({
await getVectors({
model,
input: 'x',
inputs: [textInput('x')],
headers: { 'X-Custom': 'yes' }
});
......@@ -593,7 +675,7 @@ describe('getVectorsByText function test', () => {
);
const model = buildModel({ requestUrl: 'https://custom.example/v1/embeddings' });
await getVectorsByText({ model, input: 'x' });
await getVectors({ model, inputs: [textInput('x')] });
expect(mockCreate.mock.calls[0][1]).toEqual({
path: 'https://custom.example/v1/embeddings',
......@@ -606,9 +688,9 @@ describe('getVectorsByText function test', () => {
makeResponse([[0.1, 0.2, 0.3, 0.4]], { usage: { total_tokens: 1 } })
);
await getVectorsByText({
await getVectors({
model: buildModel(),
input: 'x',
inputs: [textInput('x')],
headers: { 'X-Trace': 't1' }
});
......@@ -621,7 +703,7 @@ describe('getVectorsByText function test', () => {
mockCreate.mockResolvedValue(makeResponse([[3, 4, 0, 0]], { usage: { total_tokens: 1 } }));
const model = buildModel({ normalization: true });
const result = await getVectorsByText({ model, input: 'x' });
const result = await getVectors({ model, inputs: [textInput('x')] });
const norm = Math.sqrt(result.vectors[0].reduce((sum, v) => sum + v * v, 0));
expect(norm).toBeCloseTo(1, 10);
......@@ -632,7 +714,7 @@ describe('getVectorsByText function test', () => {
it('should reject when API response has no data', async () => {
mockCreate.mockResolvedValue({ data: null });
await expect(getVectorsByText({ model: buildModel(), input: 'x' })).rejects.toBe(
await expect(getVectors({ model: buildModel(), inputs: [textInput('x')] })).rejects.toBe(
'Embedding API is not responding'
);
});
......@@ -640,7 +722,7 @@ describe('getVectorsByText function test', () => {
it('should reject when API response data exists but has no embedding', async () => {
mockCreate.mockResolvedValue({ data: [{}] });
await expect(getVectorsByText({ model: buildModel(), input: 'x' })).rejects.toBe(
await expect(getVectors({ model: buildModel(), inputs: [textInput('x')] })).rejects.toBe(
'Embedding API is not responding'
);
});
......@@ -649,7 +731,9 @@ describe('getVectorsByText function test', () => {
const apiErr = new Error('network boom');
mockCreate.mockRejectedValue(apiErr);
await expect(getVectorsByText({ model: buildModel(), input: 'x' })).rejects.toBe(apiErr);
await expect(getVectors({ model: buildModel(), inputs: [textInput('x')] })).rejects.toBe(
apiErr
);
});
});
});
packages/service/test/core/ai/functions/queryExtension.test.ts
......@@ -73,6 +73,55 @@ describe('queryExtension', () => {
});
});
it('filters blank generated queries before cosine selection', async () => {
createLLMResponseMock.mockResolvedValue({
answerText: '[" ", " expanded query ", ""]',
requestId: 'req_query_extension_filter',
usage: {
inputTokens: 11,
outputTokens: 3,
usedUserOpenAIKey: false
}
});
await queryExtension({
query: 'original query',
histories: [],
llmModel: 'gpt-query',
embeddingModel: 'embedding-query'
});
expect(lazyGreedyQuerySelectionMock).toHaveBeenCalledWith({
originalText: 'original query',
candidates: ['expanded query'],
k: 1,
alpha: 0.3
});
});
it('does not run cosine selection when generated queries are all blank', async () => {
createLLMResponseMock.mockResolvedValue({
answerText: '[" ", ""]',
requestId: 'req_query_extension_blank',
usage: {
inputTokens: 11,
outputTokens: 3,
usedUserOpenAIKey: false
}
});
const result = await queryExtension({
query: 'original query',
histories: [],
llmModel: 'gpt-query',
embeddingModel: 'embedding-query'
});
expect(lazyGreedyQuerySelectionMock).not.toHaveBeenCalled();
expect(result.extensionQueries).toEqual([]);
expect(result.embeddingTokens).toBe(0);
});
it('splits fixed system prompt and dynamic user prompt', async () => {
filterGPTMessageByMaxContextMock.mockResolvedValue([
{
......
packages/service/test/core/ai/hooks/useTextCosine.test.ts
......@@ -5,7 +5,7 @@ import {
createMockVectorsResponse,
generateSimilarVector,
generateOrthogonalVector,
mockGetVectorsByText
mockGetVectors
} from '@test/mocks/core/ai/embedding';
describe('useTextCosine', () => {
......@@ -25,6 +25,8 @@ describe('useTextCosine', () => {
});
expect(result.selectedData).toEqual([]);
expect(result.embeddingTokens).toBe(0);
expect(mockGetVectors).not.toHaveBeenCalled();
});
it('should select k candidates when k <= candidates.length', async () => {
......@@ -73,7 +75,7 @@ describe('useTextCosine', () => {
// Create a candidate very different from original
const differentVector = generateOrthogonalVector(originalVector);
mockGetVectorsByText.mockResolvedValueOnce({
mockGetVectors.mockResolvedValueOnce({
tokens: 30,
vectors: [originalVector, differentVector, similarVector]
});
......@@ -110,7 +112,7 @@ describe('useTextCosine', () => {
expect(result.selectedData).toContain('c3');
});
it('should call getVectorsByText with correct parameters', async () => {
it('should call getVectors with correct parameters', async () => {
const { lazyGreedyQuerySelection } = useTextCosine({ embeddingModel: 'custom-model' });
await lazyGreedyQuerySelection({
originalText: 'test query',
......@@ -118,18 +120,59 @@ describe('useTextCosine', () => {
k: 1
});
expect(mockGetVectorsByText).toHaveBeenCalledWith({
expect(mockGetVectors).toHaveBeenCalledWith({
model: expect.anything(),
input: ['test query', 'candidate'],
inputs: [
{
type: 'text',
input: 'test query'
},
{
type: 'text',
input: 'candidate'
}
],
type: 'query'
});
});
it('should trim original text and skip blank candidates before embedding', async () => {
const originalVector = generateMockEmbedding('test query');
const candidateVector = generateSimilarVector(originalVector, 0.9);
mockGetVectors.mockResolvedValueOnce({
tokens: 10,
vectors: [originalVector, candidateVector]
});
const { lazyGreedyQuerySelection } = useTextCosine({ embeddingModel: 'custom-model' });
const result = await lazyGreedyQuerySelection({
originalText: ' test query ',
candidates: [' ', ' candidate ', ''],
k: 3
});
expect(mockGetVectors).toHaveBeenCalledWith({
model: expect.anything(),
inputs: [
{
type: 'text',
input: 'test query'
},
{
type: 'text',
input: 'candidate'
}
],
type: 'query'
});
expect(result.selectedData).toEqual(['candidate']);
});
it('should handle identical candidates correctly', async () => {
const originalVector = generateMockEmbedding('original');
const identicalVector = generateMockEmbedding('same');
mockGetVectorsByText.mockResolvedValueOnce({
mockGetVectors.mockResolvedValueOnce({
tokens: 30,
vectors: [originalVector, identicalVector, identicalVector, identicalVector]
});
......@@ -152,7 +195,7 @@ describe('useTextCosine', () => {
const similarVector = generateSimilarVector(originalVector, 0.9);
const differentVector = generateOrthogonalVector(originalVector);
mockGetVectorsByText.mockResolvedValueOnce({
mockGetVectors.mockResolvedValueOnce({
tokens: 25,
vectors: [originalVector, similarVector, differentVector]
});
......@@ -176,7 +219,7 @@ describe('useTextCosine', () => {
const mockResponse = createMockVectorsResponse(['test', 'candidate']);
mockResponse.tokens = 12345; // Override tokens for specific test
mockGetVectorsByText.mockResolvedValueOnce(mockResponse);
mockGetVectors.mockResolvedValueOnce(mockResponse);
const { lazyGreedyQuerySelection } = useTextCosine({
embeddingModel: 'text-embedding-ada-002'
......@@ -210,7 +253,7 @@ describe('useTextCosine', () => {
const similar2 = generateSimilarVector(similar1, 0.95); // Very close to similar1
const different = generateOrthogonalVector(originalVector);
mockGetVectorsByText.mockResolvedValueOnce({
mockGetVectors.mockResolvedValueOnce({
tokens: 40,
vectors: [originalVector, similar1, similar2, different]
});
......
packages/service/test/core/dataset/search/defaultRecall.test.ts 0 → 100644
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constants';
import { serviceEnv } from '@fastgpt/service/env';
const mockGetVectors = vi.hoisted(() => vi.fn());
const mockGetEmbeddingModel = vi.hoisted(() => vi.fn());
const mockGetDefaultRerankModel = vi.hoisted(() => vi.fn());
const mockGetLLMModel = vi.hoisted(() => vi.fn());
const mockIsImageEmbeddingModel = vi.hoisted(() => vi.fn());
const mockRecallFromVectorStore = vi.hoisted(() => vi.fn());
const mockCreateLLMResponse = vi.hoisted(() => vi.fn());
const mockMongoDatasetCollectionFind = vi.hoisted(() => vi.fn());
const mockMongoDatasetDataFind = vi.hoisted(() => vi.fn());
const mockMongoDatasetDataTextAggregate = vi.hoisted(() => vi.fn());
const mockGetImageBase64 = vi.hoisted(() => vi.fn());
const originalMultipleDataToBase64 = serviceEnv.MULTIPLE_DATA_TO_BASE64;
vi.mock('@fastgpt/service/core/ai/embedding', () => ({
getVectors: mockGetVectors
}));
vi.mock('@fastgpt/service/core/ai/model', () => ({
getEmbeddingModel: mockGetEmbeddingModel,
getDefaultRerankModel: mockGetDefaultRerankModel,
getLLMModel: mockGetLLMModel,
isImageEmbeddingModel: mockIsImageEmbeddingModel
}));
vi.mock('@fastgpt/service/common/vectorDB/controller', () => ({
recallFromVectorStore: mockRecallFromVectorStore
}));
vi.mock('@fastgpt/service/core/ai/llm/request', () => ({
createLLMResponse: mockCreateLLMResponse
}));
vi.mock('@fastgpt/service/common/file/image/utils', () => ({
getImageBase64: mockGetImageBase64
}));
vi.mock('@fastgpt/service/core/dataset/collection/schema', () => ({
DatasetColCollectionName: 'dataset_collections',
MongoDatasetCollection: {
find: mockMongoDatasetCollectionFind
}
}));
vi.mock('@fastgpt/service/core/dataset/data/schema', () => ({
DatasetDataCollectionName: 'dataset_datas',
MongoDatasetData: {
find: mockMongoDatasetDataFind
}
}));
vi.mock('@fastgpt/service/core/dataset/data/dataTextSchema', () => ({
MongoDatasetDataText: {
aggregate: mockMongoDatasetDataTextAggregate
}
}));
import { searchDatasetData } from '../../../../core/dataset/search/defaultRecall';
afterEach(() => {
serviceEnv.MULTIPLE_DATA_TO_BASE64 = originalMultipleDataToBase64;
});
describe('default recall dataset search', () => {
beforeEach(() => {
vi.clearAllMocks();
serviceEnv.MULTIPLE_DATA_TO_BASE64 = originalMultipleDataToBase64;
mockGetEmbeddingModel.mockReturnValue({
model: 'mock-embedding-model',
name: 'Mock Embedding Model'
});
mockGetDefaultRerankModel.mockReturnValue(undefined);
mockGetLLMModel.mockReturnValue({
model: 'mock-vlm-model',
name: 'Mock VLM Model',
vision: true
});
mockIsImageEmbeddingModel.mockReturnValue(false);
mockGetVectors.mockResolvedValue({
tokens: 10,
vectors: [
[0.1, 0.2],
[0.3, 0.4]
]
});
mockRecallFromVectorStore.mockResolvedValue({
results: []
});
mockMongoDatasetCollectionFind.mockImplementation((query: Record<string, any>) => {
if (query?.forbid) return [];
return {
lean: vi.fn().mockResolvedValue([])
};
});
mockMongoDatasetDataFind.mockReturnValue({
lean: vi.fn().mockResolvedValue([])
});
mockMongoDatasetDataTextAggregate.mockResolvedValue([]);
});
it('should ignore failed image caption and continue dataset search', async () => {
const userKey = { key: 'user-key', baseUrl: 'https://api.example.com/v1' };
mockCreateLLMResponse.mockRejectedValueOnce(new Error('vlm failed')).mockResolvedValueOnce({
requestId: 'req_image_caption_2',
answerText: 'red handbag on a white table',
usage: {
inputTokens: 3,
outputTokens: 2,
usedUserOpenAIKey: true
}
});
const result = await searchDatasetData({
histories: [],
teamId: 'team-1',
model: 'mock-embedding-model',
vlmModel: 'mock-vlm-model',
datasetIds: ['dataset-1'],
reRankQuery: 'black high heels',
textQueries: ['black high heels'],
imageQueries: ['data:image/png;base64,broken-image', 'data:image/png;base64,current-image'],
userKey,
limit: 5000,
searchMode: DatasetSearchModeEnum.embedding,
embeddingWeight: 0.5,
usingReRank: false
});
expect(result.imageCaptionResult).toEqual({
model: 'mock-vlm-model',
inputTokens: 3,
outputTokens: 2,
requestIds: ['req_image_caption_2'],
seconds: expect.any(Number),
usedUserOpenAIKey: true,
queries: ['red handbag on a white table']
});
expect(mockGetVectors).toHaveBeenCalledWith(
expect.objectContaining({
inputs: [
{
type: 'text',
input: 'black high heels'
},
{
type: 'text',
input: 'red handbag on a white table'
}
]
})
);
expect(mockCreateLLMResponse.mock.calls[1][0].body.messages[0].content[0].image_url.url).toBe(
'data:image/png;base64,current-image'
);
expect(mockCreateLLMResponse.mock.calls[1][0].userKey).toBe(userKey);
expect(result.searchRes).toEqual([]);
});
it('should request text and image embeddings in one getVectors call', async () => {
mockGetLLMModel.mockReturnValue(undefined);
mockIsImageEmbeddingModel.mockReturnValue(true);
mockGetVectors.mockResolvedValueOnce({
tokens: 12,
vectors: [
[0.1, 0.2],
[0.3, 0.4]
]
});
await searchDatasetData({
histories: [],
teamId: 'team-1',
model: 'mock-embedding-model',
datasetIds: ['dataset-1'],
reRankQuery: 'black high heels',
textQueries: ['black high heels'],
imageQueries: ['data:image/png;base64,current-image'],
limit: 5000,
searchMode: DatasetSearchModeEnum.embedding,
embeddingWeight: 0.5,
usingReRank: false
});
expect(mockGetVectors).toHaveBeenCalledTimes(1);
expect(mockGetVectors).toHaveBeenCalledWith(
expect.objectContaining({
inputs: [
{
type: 'text',
input: 'black high heels'
},
{
type: 'image',
input: 'data:image/png;base64,current-image'
}
]
})
);
expect(mockGetImageBase64).not.toHaveBeenCalledWith(' ');
});
it('should skip blank embedding recall inputs while preserving valid task order', async () => {
mockGetLLMModel.mockReturnValue(undefined);
mockIsImageEmbeddingModel.mockReturnValue(true);
mockGetVectors.mockResolvedValueOnce({
tokens: 12,
vectors: [
[0.1, 0.2],
[0.3, 0.4]
]
});
await searchDatasetData({
histories: [],
teamId: 'team-1',
model: 'mock-embedding-model',
datasetIds: ['dataset-1'],
reRankQuery: 'black high heels',
textQueries: [' ', ' black high heels '],
imageQueries: [' ', 'data:image/png;base64,current-image'],
limit: 5000,
searchMode: DatasetSearchModeEnum.embedding,
embeddingWeight: 0.5,
usingReRank: false
});
expect(mockGetVectors).toHaveBeenCalledWith(
expect.objectContaining({
inputs: [
{
type: 'text',
input: 'black high heels'
},
{
type: 'image',
input: 'data:image/png;base64,current-image'
}
]
})
);
});
it('should ignore failed image embedding normalization and keep text recall', async () => {
mockGetLLMModel.mockReturnValue(undefined);
mockIsImageEmbeddingModel.mockReturnValue(true);
serviceEnv.MULTIPLE_DATA_TO_BASE64 = true;
mockGetImageBase64.mockRejectedValueOnce(new Error('expired image'));
mockGetVectors.mockResolvedValueOnce({
tokens: 12,
vectors: [
[0.1, 0.2],
[0.3, 0.4]
]
});
await searchDatasetData({
histories: [],
teamId: 'team-1',
model: 'mock-embedding-model',
datasetIds: ['dataset-1'],
reRankQuery: 'black high heels',
textQueries: ['black high heels'],
imageQueries: [
'https://file.fastgpt.io/temp/team-1/expired.png?token=mock',
'data:image/png;base64,current-image'
],
limit: 5000,
searchMode: DatasetSearchModeEnum.embedding,
embeddingWeight: 0.5,
usingReRank: false
});
expect(mockGetVectors).toHaveBeenCalledWith(
expect.objectContaining({
inputs: [
{
type: 'text',
input: 'black high heels'
},
{
type: 'image',
input: 'data:image/png;base64,current-image'
}
]
})
);
});
it('should skip blank full-text queries before Mongo text search', async () => {
mockGetLLMModel.mockReturnValue(undefined);
mockIsImageEmbeddingModel.mockReturnValue(false);
const result = await searchDatasetData({
histories: [],
teamId: 'team-1',
model: 'mock-embedding-model',
datasetIds: ['dataset-1'],
reRankQuery: '',
textQueries: [' ', '\n'],
imageQueries: [],
limit: 5000,
searchMode: DatasetSearchModeEnum.fullTextRecall,
embeddingWeight: 0.5,
usingReRank: false
});
expect(mockGetVectors).not.toHaveBeenCalled();
expect(mockMongoDatasetDataTextAggregate).not.toHaveBeenCalled();
expect(result.searchRes).toEqual([]);
});
});
packages/service/test/core/dataset/search/index.test.ts 0 → 100644
import { beforeEach, describe, expect, it, vi } from 'vitest';
const mockSearchDatasetData = vi.hoisted(() => vi.fn());
const mockDatasetSearchQueryExtension = vi.hoisted(() => vi.fn());
const mockGetLLMModel = vi.hoisted(() => vi.fn());
vi.mock('@fastgpt/service/core/dataset/search/defaultRecall', () => ({
searchDatasetData: mockSearchDatasetData
}));
vi.mock('@fastgpt/service/core/dataset/search/utils', () => ({
datasetSearchQueryExtension: mockDatasetSearchQueryExtension
}));
vi.mock('@fastgpt/service/core/ai/model', () => ({
getLLMModel: mockGetLLMModel
}));
import { defaultSearchDatasetData } from '../../../../core/dataset/search';
describe('defaultSearchDatasetData', () => {
beforeEach(() => {
vi.clearAllMocks();
mockSearchDatasetData.mockResolvedValue({
searchRes: [],
embeddingTokens: 0,
reRankInputTokens: 0,
searchMode: 'embedding',
limit: 5000,
similarity: 0,
usingReRank: false,
usingSimilarityFilter: false
});
});
it('should drop whitespace-only text queries before extension and recall', async () => {
await defaultSearchDatasetData({
histories: [],
teamId: 'team-1',
datasetIds: ['dataset-1'],
model: 'embedding-model',
textQueries: [' ', '\n'],
imageQueries: ['https://files.example.com/query.png'],
limit: 5000,
datasetSearchUsingExtensionQuery: true
});
expect(mockDatasetSearchQueryExtension).not.toHaveBeenCalled();
expect(mockSearchDatasetData).toHaveBeenCalledWith(
expect.objectContaining({
reRankQuery: '',
textQueries: []
})
);
});
it('should trim text queries before query extension', async () => {
mockGetLLMModel.mockReturnValue({
model: 'query-extension-model'
});
mockDatasetSearchQueryExtension.mockResolvedValue({
searchQueries: ['first', 'second', 'expanded'],
reRankQuery: 'first\nsecond\nexpanded',
aiExtensionResult: undefined
});
await defaultSearchDatasetData({
histories: [],
teamId: 'team-1',
datasetIds: ['dataset-1'],
model: 'embedding-model',
textQueries: [' first ', ' ', 'second'],
imageQueries: [],
limit: 5000,
datasetSearchUsingExtensionQuery: true,
datasetSearchExtensionModel: 'query-extension-model'
});
expect(mockDatasetSearchQueryExtension).toHaveBeenCalledWith(
expect.objectContaining({
query: 'first\nsecond',
llmModel: 'query-extension-model'
})
);
expect(mockSearchDatasetData).toHaveBeenCalledWith(
expect.objectContaining({
reRankQuery: 'first\nsecond\nexpanded',
textQueries: ['first', 'second', 'expanded']
})
);
});
});
packages/service/test/core/dataset/search/utils.test.ts
import { describe, it, expect } from 'vitest';
import { computeFilterIntersection } from '@fastgpt/service/core/dataset/search/utils';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { serviceEnv } from '@fastgpt/service/env';
describe('computeFilterIntersection', () => {
describe('edge cases', () => {
it('should return undefined for empty array', () => {
const result = computeFilterIntersection([]);
expect(result).toBeUndefined();
});
const mockQueryExtension = vi.hoisted(() => vi.fn());
const mockGetImageBase64 = vi.hoisted(() => vi.fn());
it('should return undefined for all undefined arrays', () => {
const result = computeFilterIntersection([undefined, undefined, undefined]);
expect(result).toBeUndefined();
});
vi.mock('@fastgpt/service/core/ai/functions/queryExtension', () => ({
queryExtension: mockQueryExtension
}));
it('should return single array as-is', () => {
const result = computeFilterIntersection([['a', 'b', 'c']]);
expect(result).toEqual(['a', 'b', 'c']);
});
vi.mock('@fastgpt/service/common/file/image/utils', () => ({
getImageBase64: mockGetImageBase64
}));
it('should filter out undefined arrays', () => {
const result = computeFilterIntersection([undefined, ['a', 'b'], undefined]);
expect(result).toEqual(['a', 'b']);
});
import {
computeFilterIntersection,
datasetSearchQueryExtension,
normalizeImageToBase64
} from '../../../../core/dataset/search/utils';
it('should return empty array when intersection is empty', () => {
const result = computeFilterIntersection([
['a', 'b'],
['c', 'd']
]);
expect(result).toEqual([]);
});
const originalMultipleDataToBase64 = serviceEnv.MULTIPLE_DATA_TO_BASE64;
afterEach(() => {
serviceEnv.MULTIPLE_DATA_TO_BASE64 = originalMultipleDataToBase64;
});
describe('normalizeImageToBase64', () => {
beforeEach(() => {
vi.clearAllMocks();
serviceEnv.MULTIPLE_DATA_TO_BASE64 = originalMultipleDataToBase64;
});
describe('two arrays', () => {
it('should compute intersection of two arrays', () => {
const result = computeFilterIntersection([
['a', 'b', 'c'],
['b', 'c', 'd']
]);
expect(result).toEqual(['b', 'c']);
});
it('should keep image url unchanged when base64 conversion is disabled', async () => {
serviceEnv.MULTIPLE_DATA_TO_BASE64 = false;
it('should handle duplicate elements', () => {
const result = computeFilterIntersection([
['a', 'a', 'b', 'b'],
['a', 'b', 'c']
]);
expect(result).toEqual(['a', 'a', 'b', 'b']);
});
const result = await normalizeImageToBase64('https://example.com/image.png');
it('should preserve order from first array', () => {
const result = computeFilterIntersection([
['c', 'b', 'a'],
['a', 'b', 'c']
]);
expect(result).toEqual(['c', 'b', 'a']);
});
expect(result).toBe('https://example.com/image.png');
expect(mockGetImageBase64).not.toHaveBeenCalled();
});
describe('three arrays (tags, createTime, collectionIds)', () => {
it('should compute intersection of three arrays', () => {
const tagIds = ['id1', 'id2', 'id3'];
const timeIds = ['id2', 'id3', 'id4'];
const collectionIds = ['id3', 'id4', 'id5'];
it('should keep internal object keys unchanged when base64 conversion is disabled', async () => {
serviceEnv.MULTIPLE_DATA_TO_BASE64 = false;
const keys = ['dataset/team/file.png', 'temp/team/file.png', 'chat/app/user/chat/file.png'];
const result = computeFilterIntersection([tagIds, timeIds, collectionIds]);
expect(result).toEqual(['id3']);
await expect(Promise.all(keys.map((key) => normalizeImageToBase64(key)))).resolves.toEqual([
'dataset/team/file.png',
'temp/team/file.png',
'chat/app/user/chat/file.png'
]);
expect(mockGetImageBase64).not.toHaveBeenCalled();
});
it('should convert image url to base64 when base64 conversion is enabled', async () => {
serviceEnv.MULTIPLE_DATA_TO_BASE64 = true;
mockGetImageBase64.mockResolvedValue({
completeBase64: 'data:image/png;base64,converted'
});
it('should return empty when no common elements', () => {
const tagIds = ['id1', 'id2'];
const timeIds = ['id3', 'id4'];
const collectionIds = ['id5', 'id6'];
const result = await normalizeImageToBase64('https://example.com/image.png');
expect(result).toBe('data:image/png;base64,converted');
expect(mockGetImageBase64).toHaveBeenCalledWith('https://example.com/image.png');
});
const result = computeFilterIntersection([tagIds, timeIds, collectionIds]);
expect(result).toEqual([]);
it('should treat internal object keys as plain inputs when base64 conversion is enabled', async () => {
serviceEnv.MULTIPLE_DATA_TO_BASE64 = true;
mockGetImageBase64.mockResolvedValue({
completeBase64: 'data:image/png;base64,converted'
});
it('should handle partial undefined', () => {
const tagIds = ['id1', 'id2', 'id3'];
const collectionIds = ['id2', 'id3', 'id4'];
const result = await normalizeImageToBase64('dataset/team/file.png');
expect(result).toBe('data:image/png;base64,converted');
expect(mockGetImageBase64).toHaveBeenCalledWith('dataset/team/file.png');
});
it('should keep data image unchanged regardless of base64 conversion flag', async () => {
serviceEnv.MULTIPLE_DATA_TO_BASE64 = false;
const result = await normalizeImageToBase64('data:image/png;base64,input');
const result = computeFilterIntersection([tagIds, undefined, collectionIds]);
expect(result).toEqual(['id2', 'id3']);
expect(result).toBe('data:image/png;base64,input');
expect(mockGetImageBase64).not.toHaveBeenCalled();
});
});
describe('computeFilterIntersection', () => {
it('should return undefined when every filter is absent', () => {
expect(computeFilterIntersection([])).toBeUndefined();
expect(computeFilterIntersection([undefined, undefined])).toBeUndefined();
});
it('should compute intersection while ignoring absent filters', () => {
expect(
computeFilterIntersection([['tag_1', 'tag_2', 'tag_3'], undefined, ['tag_2', 'tag_3']])
).toEqual(['tag_2', 'tag_3']);
});
it('should return empty array when active filters have no overlap', () => {
expect(computeFilterIntersection([['tag_1', 'tag_2'], ['tag_3'], ['tag_4']])).toEqual([]);
});
});
describe('datasetSearchQueryExtension', () => {
beforeEach(() => {
vi.clearAllMocks();
});
it('should treat combined text queries as one plain query string', async () => {
mockQueryExtension.mockResolvedValue({
rawQuery: 'first\nsecond',
extensionQueries: ['first extension', 'second extension'],
llmModel: 'mock-llm',
embeddingModel: 'mock-embedding',
requestId: 'req-1',
seconds: 1,
inputTokens: 10,
outputTokens: 5,
usedUserOpenAIKey: false,
embeddingTokens: 6
});
it('should handle all same elements', () => {
const ids = ['id1', 'id2', 'id3'];
const result = computeFilterIntersection([ids, ids, ids]);
expect(result).toEqual(['id1', 'id2', 'id3']);
const result = await datasetSearchQueryExtension({
query: 'first\nsecond',
llmModel: 'mock-llm',
embeddingModel: 'mock-embedding',
histories: []
});
expect(mockQueryExtension).toHaveBeenCalledWith(
expect.objectContaining({
query: 'first\nsecond'
})
);
expect(result.searchQueries).toEqual(['first\nsecond', 'first extension', 'second extension']);
expect(result.reRankQuery).toBe('first\nsecond\nfirst extension\nsecond extension');
});
describe('performance with Set optimization', () => {
it('should handle large arrays efficiently', () => {
const size = 10000;
const arr1 = Array.from({ length: size }, (_, i) => `id${i}`);
const arr2 = Array.from({ length: size }, (_, i) => `id${i + size / 2}`);
it('should not parse json-like query strings as pre-expanded query arrays', async () => {
mockQueryExtension.mockResolvedValue({
rawQuery: '["first","second"]',
extensionQueries: ['first extension'],
llmModel: 'mock-llm',
embeddingModel: 'mock-embedding',
requestId: 'req-1',
seconds: 1,
inputTokens: 10,
outputTokens: 5,
usedUserOpenAIKey: false,
embeddingTokens: 6
});
const result = await datasetSearchQueryExtension({
query: '["first","second"]',
llmModel: 'mock-llm',
embeddingModel: 'mock-embedding',
histories: []
});
const start = performance.now();
const result = computeFilterIntersection([arr1, arr2]);
const duration = performance.now() - start;
expect(mockQueryExtension).toHaveBeenCalledWith(
expect.objectContaining({
query: '["first","second"]'
})
);
expect(result.searchQueries).toEqual(['["first","second"]', 'first extension']);
});
expect(result?.length).toBe(size / 2);
expect(duration).toBeLessThan(100); // Should complete within 100ms
it('should ignore blank extension queries before recall', async () => {
mockQueryExtension.mockResolvedValue({
rawQuery: 'first',
extensionQueries: [' ', 'first extension', '\n'],
llmModel: 'mock-llm',
embeddingModel: 'mock-embedding',
requestId: 'req-1',
seconds: 1,
inputTokens: 10,
outputTokens: 5,
usedUserOpenAIKey: false,
embeddingTokens: 6
});
const result = await datasetSearchQueryExtension({
query: 'first',
llmModel: 'mock-llm',
embeddingModel: 'mock-embedding',
histories: []
});
expect(result.searchQueries).toEqual(['first', 'first extension']);
expect(result.reRankQuery).toBe('first\nfirst extension');
});
});
packages/service/test/core/workflow/dispatch/ai/agent/sub/dataset.test.ts
......@@ -16,7 +16,7 @@ const {
formatModelChars2PointsMock: vi.fn()
}));
vi.mock('@fastgpt/service/core/dataset/search/controller', () => ({
vi.mock('@fastgpt/service/core/dataset/search', () => ({
defaultSearchDatasetData: defaultSearchDatasetDataMock
}));
......
packages/service/test/core/workflow/dispatch/ai/toolcall/hooks/useToolRunner.test.ts
import { beforeEach, describe, expect, it, vi } from 'vitest';
import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
import { useToolRunner } from '@fastgpt/service/core/workflow/dispatch/ai/toolcall/hooks/useToolRunner';
......@@ -78,12 +79,14 @@ const createRunner = ({
getToolInfo,
runtimeNodes = [],
runtimeEdges = [],
allFiles = new Map()
allFiles = new Map(),
fileUrls = []
}: {
getToolInfo: (name: string) => any;
runtimeNodes?: any[];
runtimeEdges?: any[];
allFiles?: Map<string, any>;
fileUrls?: string[];
}) => {
const cacheToolFlowResponse = vi.fn();
const appendToolFlowResponse = vi.fn();
......@@ -93,6 +96,7 @@ const createRunner = ({
runtimeNodes,
runtimeEdges,
allFiles,
fileUrls,
getToolInfo,
cacheToolFlowResponse,
appendToolFlowResponse,
......@@ -254,6 +258,92 @@ describe('useToolRunner', () => {
});
});
it('injects parent file urls into dataset search tool calls', async () => {
const runtimeNodes = [
{
nodeId: 'dataset_search',
inputs: [
{
key: NodeInputKeyEnum.userChatInput,
value: 'legacy default'
},
{
key: NodeInputKeyEnum.datasetSearchInput,
value: []
},
{
key: 'limit',
value: 10
}
]
}
];
const runtimeEdges = [
{
target: 'dataset_search'
}
];
runWorkflowMock.mockResolvedValue({
toolResponses: 'dataset ok',
assistantResponses: [],
flowUsages: [],
workflowInteractiveResponse: undefined,
flowResponses: []
});
const { runTool } = createRunner({
runtimeNodes,
runtimeEdges,
fileUrls: ['https://files/image.png'],
getToolInfo: () => ({
type: 'user',
name: 'Dataset search',
avatar: 'dataset-avatar',
rawData: {
nodeId: 'dataset_search',
flowNodeType: FlowNodeTypeEnum.datasetSearchNode
}
})
});
const call = createCall({
id: 'call_dataset_search',
name: 'dataset_search',
args: '{"datasetSearchInput":"red shoes","limit":3}'
});
const result = await runTool({ call });
expect(runtimeNodes[0]).toEqual({
nodeId: 'dataset_search',
isEntry: true,
inputs: [
{
key: NodeInputKeyEnum.userChatInput,
value: ''
},
{
key: NodeInputKeyEnum.datasetSearchInput,
value: ['red shoes', 'https://files/image.png']
},
{
key: 'limit',
value: 3
}
]
});
expect(runtimeEdges[0]).toEqual({
target: 'dataset_search',
status: 'active'
});
expect(result).toEqual({
response: 'dataset ok',
assistantMessages: [],
usages: [],
interactive: undefined,
stop: false
});
});
it('runs user workflow tools and interactive resume paths', async () => {
const usage = {
moduleName: 'tool',
......
packages/service/test/core/workflow/dispatch/dataset/search.test.ts
......@@ -5,19 +5,21 @@ import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
const {
defaultSearchDatasetDataMock,
deepRagSearchMock,
findDatasetByIdMock,
formatModelChars2PointsMock,
usagePushMock
} = vi.hoisted(() => ({
defaultSearchDatasetDataMock: vi.fn(),
deepRagSearchMock: vi.fn(),
findDatasetByIdMock: vi.fn(),
formatModelChars2PointsMock: vi.fn(),
usagePushMock: vi.fn()
}));
vi.mock('@fastgpt/service/core/dataset/search/controller', () => ({
vi.mock('@fastgpt/service/core/dataset/search', () => ({
defaultSearchDatasetData: defaultSearchDatasetDataMock,
deepRagSearch: vi.fn()
deepRagSearch: deepRagSearchMock
}));
vi.mock('@fastgpt/service/core/dataset/schema', () => ({
......@@ -208,4 +210,123 @@ describe('dispatchDatasetSearch', () => {
])
);
});
it('adds image caption request ids and skips platform points when external OpenAI key is used', async () => {
const userKey = { key: 'user-key', baseUrl: 'https://api.example.com/v1' };
defaultSearchDatasetDataMock.mockResolvedValue({
searchRes: [],
embeddingTokens: 20,
reRankInputTokens: 0,
usingSimilarityFilter: true,
usingReRank: false,
imageCaptionResult: {
model: 'gpt-vision',
inputTokens: 4,
outputTokens: 3,
requestIds: ['req_image_caption_1', 'req_image_caption_2'],
seconds: 1.5,
usedUserOpenAIKey: true,
queries: ['red handbag', 'blue sneaker']
}
});
const result = await dispatchDatasetSearch({
runningAppInfo: { teamId: 'team_1' },
runningUserInfo: { tmbId: 'tmb_1' },
externalProvider: {
openaiAccount: userKey
},
histories: [],
node: { name: 'Dataset Search' },
params: {
datasets: [{ datasetId: 'dataset_1' }],
similarity: 0.4,
limit: 5000,
userChatInput: '',
datasetSearchInput: ['https://files.example.com/query.png'],
authTmbId: false,
searchMode: DatasetSearchModeEnum.embedding,
embeddingWeight: 0.5,
usingReRank: false,
rerankWeight: 0.5,
datasetSearchUsingExtensionQuery: false
},
usagePush: usagePushMock
} as any);
expect(defaultSearchDatasetDataMock).toHaveBeenCalledWith(
expect.objectContaining({
userKey,
imageQueries: ['https://files.example.com/query.png']
})
);
const nodeResponse = result[DispatchNodeResponseKeyEnum.nodeResponse];
expect(nodeResponse?.childrenResponses?.[0]).toMatchObject({
id: 'req_image_caption_1',
nodeId: 'req_image_caption_1',
moduleType: FlowNodeTypeEnum.datasetSearchNode,
moduleName: 'account_usage:image_parse',
moduleLogo: 'core/workflow/template/datasetSearch',
runningTime: 1.5,
model: 'gpt-vision name',
llmRequestIds: ['req_image_caption_1', 'req_image_caption_2'],
inputTokens: 4,
outputTokens: 3,
totalPoints: 0,
textOutput: 'red handbag\nblue sneaker'
});
expect(usagePushMock).toHaveBeenCalledWith(
expect.arrayContaining([
expect.objectContaining({
moduleName: 'account_usage:image_parse',
totalPoints: 0
})
])
);
});
it('uses default recall for image-only input even when deep search is enabled', async () => {
defaultSearchDatasetDataMock.mockResolvedValue({
searchRes: [],
embeddingTokens: 4,
reRankInputTokens: 0,
usingSimilarityFilter: false,
usingReRank: false
});
const result = await dispatchDatasetSearch({
runningAppInfo: { teamId: 'team_1' },
runningUserInfo: { tmbId: 'tmb_1' },
externalProvider: {},
histories: [],
node: { name: 'Dataset Search' },
params: {
datasets: [{ datasetId: 'dataset_1' }],
similarity: 0.4,
limit: 5000,
userChatInput: '',
datasetSearchInput: ['https://files.example.com/query.png'],
authTmbId: false,
searchMode: DatasetSearchModeEnum.embedding,
embeddingWeight: 0.5,
usingReRank: false,
rerankWeight: 0.5,
datasetSearchUsingExtensionQuery: false,
datasetDeepSearch: true
},
usagePush: usagePushMock
} as any);
expect(deepRagSearchMock).not.toHaveBeenCalled();
expect(defaultSearchDatasetDataMock).toHaveBeenCalledWith(
expect.objectContaining({
textQueries: [],
imageQueries: ['https://files.example.com/query.png']
})
);
expect(result[DispatchNodeResponseKeyEnum.nodeResponse]?.datasetQueries).toEqual([
'https://files.example.com/query.png'
]);
});
});
packages/service/test/core/workflow/dispatch/dataset/utils.test.ts 0 → 100644
import { describe, expect, it } from 'vitest';
import { normalizeDatasetSearchInput } from '../../../../../core/workflow/dispatch/dataset/utils';
describe('normalizeDatasetSearchInput', () => {
it('should split text queries and http image urls', () => {
const result = normalizeDatasetSearchInput([
' black high heels ',
'temp/team-1/search-image.png',
'chat/team-1/manual.pdf',
'https://example.com/current.png',
'data:image/png;base64,abc',
'dataset/team-1/photo.webp',
'https://example.com/manual.pdf',
''
]);
expect(result.textQueries).toEqual([
'black high heels',
'temp/team-1/search-image.png',
'chat/team-1/manual.pdf',
'data:image/png;base64,abc',
'dataset/team-1/photo.webp'
]);
expect(result.imageQueries).toEqual(['https://example.com/current.png']);
});
it('should only classify http image urls by parsed file type', () => {
const result = normalizeDatasetSearchInput([
'/api/file/read?filename=current.png',
'/api/file/read?filename=manual.pdf',
'https://cdn.example.com/download?filename=current.png',
'https://cdn.example.com/download?filename=current.png'
]);
expect(result.textQueries).toEqual([
'/api/file/read?filename=current.png',
'/api/file/read?filename=manual.pdf'
]);
expect(result.imageQueries).toEqual(['https://cdn.example.com/download?filename=current.png']);
});
it('should remove duplicated text queries and images', () => {
const result = normalizeDatasetSearchInput([
'same query',
'same query',
'https://example.com/a.png',
'https://example.com/a.png'
]);
expect(result.textQueries).toEqual(['same query']);
expect(result.imageQueries).toEqual(['https://example.com/a.png']);
});
});
packages/service/test/core/workflow/dispatch/index.test.ts
......@@ -559,7 +559,7 @@ describe('WorkflowQueue', () => {
const group1 = [createEdge('A', 'D', 'waiting')];
const group2 = [createEdge('B', 'D', 'skipped'), createEdge('C', 'D', 'skipped')];
const nodeEdgeGroupsMap = new Map([['D', [group1, group2]]]);
console.log(nodeEdgeGroupsMap);
const result = WorkflowQueue.getNodeRunStatus({ node, nodeEdgeGroupsMap });
expect(result).toBe('wait');
......
packages/service/test/core/workflow/utils/context.test.ts
......@@ -3,15 +3,21 @@ import {
parseUrlToFileType,
runWithContext,
getWorkflowContext,
updateWorkflowContextVal,
WorkflowContext
} from '@fastgpt/service/core/workflow/utils/context';
updateWorkflowContextVal
} from '../../../../core/workflow/utils/context';
import { ChatFileTypeEnum } from '@fastgpt/global/core/chat/constants';
const createWorkflowContext = (queryUrlTypeMap: Record<string, ChatFileTypeEnum>) => ({
queryUrlTypeMap,
mcpClientMemory: {}
});
describe('WorkflowContext', () => {
describe('runWithContext / getWorkflowContext', () => {
it('should provide context inside callback', () => {
const ctx = { queryUrlTypeMap: { 'http://a.com/f.pdf': ChatFileTypeEnum.file } };
const ctx = createWorkflowContext({
'http://a.com/f.pdf': ChatFileTypeEnum.file
});
runWithContext(ctx, () => {
const store = getWorkflowContext();
......@@ -25,8 +31,8 @@ describe('WorkflowContext', () => {
});
it('should isolate nested contexts', () => {
const outer = { queryUrlTypeMap: { a: ChatFileTypeEnum.file } };
const inner = { queryUrlTypeMap: { b: ChatFileTypeEnum.image } };
const outer = createWorkflowContext({ a: ChatFileTypeEnum.file });
const inner = createWorkflowContext({ b: ChatFileTypeEnum.image });
runWithContext(outer, () => {
expect(getWorkflowContext()?.queryUrlTypeMap).toEqual(outer.queryUrlTypeMap);
......@@ -41,7 +47,7 @@ describe('WorkflowContext', () => {
});
it('should work with async functions', async () => {
const ctx = { queryUrlTypeMap: { url1: ChatFileTypeEnum.image } };
const ctx = createWorkflowContext({ url1: ChatFileTypeEnum.image });
await new Promise<void>((resolve) => {
runWithContext(ctx, async () => {
......@@ -55,7 +61,7 @@ describe('WorkflowContext', () => {
describe('updateWorkflowContextVal', () => {
it('should update existing context values', () => {
const ctx = { queryUrlTypeMap: { a: ChatFileTypeEnum.file } };
const ctx = createWorkflowContext({ a: ChatFileTypeEnum.file });
runWithContext(ctx, () => {
updateWorkflowContextVal({
......@@ -75,7 +81,7 @@ describe('WorkflowContext', () => {
});
it('should support partial updates', () => {
const ctx = { queryUrlTypeMap: { a: ChatFileTypeEnum.file } };
const ctx = createWorkflowContext({ a: ChatFileTypeEnum.file });
runWithContext(ctx, () => {
// Update with empty partial — no keys iterated
......@@ -88,7 +94,7 @@ describe('WorkflowContext', () => {
describe('parseUrlToFileType with context', () => {
it('should use queryUrlTypeMap to determine file type', () => {
const url = 'https://example.com/unknown-resource';
const ctx = { queryUrlTypeMap: { [url]: ChatFileTypeEnum.image } };
const ctx = createWorkflowContext({ [url]: ChatFileTypeEnum.image });
runWithContext(ctx, () => {
const result = parseUrlToFileType(url);
......@@ -98,7 +104,7 @@ describe('WorkflowContext', () => {
it('should prefer context type over extension-based detection', () => {
const url = 'https://example.com/photo.png';
const ctx = { queryUrlTypeMap: { [url]: ChatFileTypeEnum.file } };
const ctx = createWorkflowContext({ [url]: ChatFileTypeEnum.file });
runWithContext(ctx, () => {
const result = parseUrlToFileType(url);
......@@ -108,7 +114,7 @@ describe('WorkflowContext', () => {
});
it('should fall back to extension detection when URL not in context', () => {
const ctx = { queryUrlTypeMap: { 'other-url': ChatFileTypeEnum.file } };
const ctx = createWorkflowContext({ 'other-url': ChatFileTypeEnum.file });
runWithContext(ctx, () => {
const result = parseUrlToFileType('https://example.com/photo.png');
......@@ -339,21 +345,21 @@ describe('parseUrlToFileType', () => {
describe('edge cases', () => {
it('should return undefined for non-string input', () => {
// @ts-ignore - testing runtime behavior
// @ts-expect-error testing runtime behavior
const result = parseUrlToFileType(123);
expect(result).toBeUndefined();
});
it('should return undefined for null', () => {
// @ts-ignore - testing runtime behavior
// @ts-expect-error testing runtime behavior
const result = parseUrlToFileType(null);
expect(result).toBeUndefined();
});
it('should return undefined for undefined', () => {
// @ts-ignore - testing runtime behavior
// @ts-expect-error testing runtime behavior
const result = parseUrlToFileType(undefined);
expect(result).toBeUndefined();
......
packages/service/vitest.config.ts
......@@ -23,7 +23,7 @@ export default defineConfig({
},
coverage: {
enabled: true,
reporter: ['text', 'text-summary', 'html', 'json-summary', 'json'],
reporter: ['html', 'json-summary', 'json'],
reportOnFailure: true,
include: ['common/**/*.ts', 'core/**/*.ts', 'support/**/*.ts', 'worker/**/*.ts'],
exclude: [
......
packages/web/components/common/Image/PhotoView.tsx
import React from 'react';
import { PhotoProvider, PhotoView } from 'react-photo-view';
import { PhotoProvider, PhotoSlider, PhotoView } from 'react-photo-view';
import 'react-photo-view/dist/react-photo-view.css';
import { type ImageProps } from '@chakra-ui/react';
import { useSystem } from '../../../hooks/useSystem';
import Loading from '../MyLoading';
import MyImage from './MyImage';
type MyPhotoSliderProps = {
src?: string;
visible: boolean;
onClose: () => void;
imageKey?: string;
};
const MyPhotoView = (props: ImageProps) => {
const { isPc } = useSystem();
......@@ -23,4 +30,20 @@ const MyPhotoView = (props: ImageProps) => {
);
};
export const MyPhotoSlider = ({ src, visible, onClose, imageKey }: MyPhotoSliderProps) => {
const { isPc } = useSystem();
return (
<PhotoSlider
images={src ? [{ key: imageKey || src, src }] : []}
visible={visible}
onClose={onClose}
maskOpacity={0.6}
bannerVisible={!isPc}
photoClosable
loadingElement={<Loading fixed={false} />}
/>
);
};
export default MyPhotoView;
packages/web/i18n/en/workflow.json
......@@ -14,6 +14,10 @@
"Variable_name": "Variable name",
"add_new_input": "add_new Input",
"add_new_output": "New output",
"agent.prompt_skill_tip": "@ to select Skills and tools, \"/\" to select variables",
"agent.select_skill": "Select Skill",
"agent.select_tool": "Select Tool",
"agent.tools": "Tools",
"append_application_reply_to_history_as_new_context": "Append the application's reply to the history as new context",
"application_call": "Application Call",
"assigned_reply": "Assigned Reply",
......@@ -21,6 +25,7 @@
"auth_tmb_id_tip": "After it is turned on, when the application is released to the outside world, the knowledge base will be filtered based on whether the user has permission to the knowledge base.\n\nIf it is not enabled, the configured knowledge base will be searched directly without permission filtering.",
"auto_align": "Automatic alignment",
"can_not_loop": "This node can't loop.",
"can_not_parallel": "This node cannot be used inside a Parallel Run node",
"choose_another_application_to_call": "Select another application to call",
"classification_result": "Classification Result",
"click_to_change_reference": "Click to switch input mode",
......@@ -41,9 +46,15 @@
"confirm_delete_field_tip": "Confirm delete this field?",
"contains": "Contains",
"content_to_retrieve": "Content to Retrieve",
"content_to_search": "Content to Search",
"content_to_search": "Search term or search statement",
"context_menu.add_comment": "Add comment",
"create_link_error": "Error creating link",
"current_index": "Current index",
"current_index_desc": "0-based index of the current element in array mode.",
"current_item": "Current item",
"current_item_desc": "Element being processed in the current iteration (array mode).",
"current_iteration": "Current loop count",
"current_iteration_desc": "1-based iteration counter in conditional mode.",
"custom_feedback": "Custom Feedback",
"custom_input": "Custom variables",
"dataset_quote_role": "Role",
......@@ -95,6 +106,8 @@
"intro_knowledge_base_search_merge": "Can merge multiple Dataset search results for output. Uses RRF merging method for final sorting output.",
"intro_laf_function_call": "Can call cloud functions under the Laf account.",
"intro_loop": "Input an array, iterate through the array and use each array element as an input element to execute the workflow.",
"intro_loop_run": "Run a repeating sub-workflow; the loop's outputs decide whether to continue into the next iteration.",
"intro_parallel_run": "Input an array and run the workflow for each element in parallel, then aggregate the results.",
"intro_plugin_input": "Can configure what inputs the plugin needs and use these inputs to run the plugin.",
"intro_question_classification": "Determine the type of question based on the user's history and current question. Multiple question types can be added. Below is a template example:\nType 1: Greeting\nType 2: Questions about product 'usage'\nType 3: Questions about product 'purchase'\nType 4: Other questions",
"intro_question_optimization": "Using question optimization can improve the accuracy of Dataset searches during continuous conversations. After using this function, AI will first construct one or more new search terms based on the context, which are more conducive to Dataset searches. This module is already built into the Dataset search module. If you only perform a single Dataset search, you can directly use the built-in completion function of the Dataset.",
......@@ -121,53 +134,28 @@
"less_than_or_equal_to": "Less Than or Equal To",
"loop": "Batch Run",
"loop_body": "loop body",
"loop_custom_outputs": "Custom outputs",
"loop_custom_outputs_tip": "Declare output fields exposed by the loop. Each field references an output of a node inside the sub-workflow.",
"loop_end": "End",
"loop_input_array": "array",
"loop_result": "Array execution results",
"loop_start": "Start",
"parallel_run": "Parallel Run",
"intro_parallel_run": "Input an array and run the workflow for each element in parallel, then aggregate the results.",
"parallel_run_execution_logic": "Execution Logic",
"parallel_run_end_intro": "Select a variable as the output of the parallel run",
"parallel_run_max_concurrency": "Max Concurrency",
"parallel_run_max_concurrency_tip": "Maximum number of tasks running in parallel (range: 1 to the upper limit, default 5).",
"parallel_run_max_retry_times": "Retry Times on Error",
"parallel_run_max_retry_times_tip": "Maximum number of retries for a failed task (range: 0–5, default 3). Set to 0 to disable retries.",
"can_not_parallel": "This node cannot be used inside a Parallel Run node",
"parallel_success_results": "Success Results",
"parallel_success_results_desc": "Outputs of all successful tasks, in input order (successful items only).",
"parallel_full_results": "Full Results",
"parallel_full_results_desc": "Array of the same length as the input, each item is {success, message, data}: on success success=true, message is empty, data is the output; on failure success=false, message contains the error, data is null.",
"parallel_status": "Completion Status",
"parallel_status_desc": "Overall run status: success, partial_success, or failed.",
"parallel_task": "Task {{index}}",
"parallel_task_not_reach_end": "Sub-workflow did not reach the end node",
"parallel_task_interactive_not_supported": "Sub-workflow triggered an interactive response, which is not supported in parallel runs",
"loop_run": "Loop",
"intro_loop_run": "Run a repeating sub-workflow; the loop's outputs decide whether to continue into the next iteration.",
"loop_run_mode": "Loop type",
"loop_run_mode_tip": "Choose how the loop is driven: iterate over array elements, or run until a Loop break node is hit.",
"loop_run_mode_array": "Array loop",
"loop_run_mode_array_desc": "Process each item in the list in order",
"loop_run_mode_conditional": "Conditional loop",
"loop_run_mode_conditional_desc": "Keep running until a specific condition is met",
"loop_run_input_array": "Array",
"loop_custom_outputs": "Custom outputs",
"loop_custom_outputs_tip": "Declare output fields exposed by the loop. Each field references an output of a node inside the sub-workflow.",
"loop_run_break": "Loop break",
"loop_run_break_tip": "This module must be used inside a loop node. When executed, it will force the loop node to stop running.",
"loop_run_break_must_inside_loop_run": "Loop break node can only be placed inside a Loop container",
"loop_run_break_tip": "This module must be used inside a loop node. When executed, it will force the loop node to stop running.",
"loop_run_conditional_requires_break": "Conditional loop must contain at least one Loop break node",
"loop_run_input_array": "Array",
"loop_run_input_not_array": "Loop input is not an array",
"loop_run_iteration_failed": "Loop iteration failed",
"loop_run_max_iterations_exceeded": "Loop reached maximum iteration limit",
"loop_run_mode": "Loop type",
"loop_run_mode_array": "Array loop",
"loop_run_mode_array_desc": "Process each item in the list in order",
"loop_run_mode_conditional": "Conditional loop",
"loop_run_mode_conditional_desc": "Keep running until a specific condition is met",
"loop_run_mode_tip": "Choose how the loop is driven: iterate over array elements, or run until a Loop break node is hit.",
"loop_run_start": "Loop start",
"current_index": "Current index",
"current_index_desc": "0-based index of the current element in array mode.",
"current_item": "Current item",
"current_item_desc": "Element being processed in the current iteration (array mode).",
"current_iteration": "Current loop count",
"current_iteration_desc": "1-based iteration counter in conditional mode.",
"loop_start": "Start",
"max_dialog_rounds": "Maximum Number of Dialog Rounds",
"max_tokens": "Maximum Tokens",
"mouse_priority": "Mouse first\n- Press the left button to drag the canvas\n- Hold down Shift and drag to box-select multiple nodes",
......@@ -179,6 +167,23 @@
"optional_value_type": "Optional Value Type",
"optional_value_type_tip": "You can specify one or more data types. When dynamically adding fields, users can only select the configured types.",
"pan_priority": "Touchpad first\n- Drag and drop with one finger to select nodes\n- Swipe with two fingers to move the canvas\n- Use three fingers to move nodes quickly",
"parallel_full_results": "Full Results",
"parallel_full_results_desc": "Array of the same length as the input, each item is {success, message, data}: on success success=true, message is empty, data is the output; on failure success=false, message contains the error, data is null.",
"parallel_run": "Parallel Run",
"parallel_run_end_intro": "Select a variable as the output of the parallel run",
"parallel_run_execution_logic": "Execution Logic",
"parallel_run_max_concurrency": "Max Concurrency",
"parallel_run_max_concurrency_tip": "Maximum number of tasks running in parallel (range: 1 to the upper limit, default 5).",
"parallel_run_max_retry_times": "Retry Times on Error",
"parallel_run_max_retry_times_tip": "Maximum number of retries for a failed task (range: 0–5, default 3). Set to 0 to disable retries.",
"parallel_status": "Completion Status",
"parallel_status_desc": "Overall run status: success, partial_success, or failed.",
"parallel_success_results": "Success Results",
"parallel_success_results_desc": "Outputs of all successful tasks, in input order (successful items only).",
"parallel_task": "Task {{index}}",
"parallel_task_interactive_not_supported": "Sub-workflow triggered an interactive response, which is not supported in parallel runs",
"parallel_task_not_reach_end": "Sub-workflow did not reach the end node",
"params_setting": "Parameters",
"pass_returned_object_as_output_to_next_nodes": "Pass the object returned in the code as output to the next nodes. The variable name needs to correspond to the return key.",
"please_enter_node_name": "Enter the node name",
"plugin.Instruction_Tip": "You can configure an instruction to explain the purpose of the plugin. This instruction will be displayed each time the plugin is used. Supports standard Markdown syntax.",
......@@ -205,6 +210,7 @@
"response.Error": "Error",
"response.Read file result": "Read File Result",
"response.read files": "Read Files",
"search_query": "Search content",
"select_an_application": "Select an Application",
"select_another_application_to_call": "You can choose another application to call",
"select_default_option": "Select the default value",
......@@ -215,10 +221,6 @@
"template.agent_intro": "Automatically select one or more functional blocks for calling through the AI model, or call plugins.",
"template.agent_module": "Agent",
"template.agent_module_intro": "Associate required tools and Skills to enable AI-driven autonomous invocation and planning",
"agent.tools": "Tools",
"agent.prompt_skill_tip": "@ to select Skills and tools, \"/\" to select variables",
"agent.select_skill": "Select Skill",
"agent.select_tool": "Select Tool",
"template.ai_chat": "AI Chat",
"template.ai_chat_intro": "AI Large Model Chat",
"template.dataset_search": "Dataset Search",
......@@ -271,6 +273,5 @@
"workflow.My edit": "My Edit",
"workflow.Switch_success": "Switch Successful",
"workflow.Team cloud": "Team Cloud",
"workflow.exit_tips": "Your changes have not been saved. 'Exit directly' will not save your edits.",
"params_setting": "Parameters"
"workflow.exit_tips": "Your changes have not been saved. 'Exit directly' will not save your edits."
}
packages/web/i18n/zh-CN/workflow.json
......@@ -14,6 +14,10 @@
"Variable_name": "变量名",
"add_new_input": "新增输入",
"add_new_output": "新增输出",
"agent.prompt_skill_tip": "@选择 Skill 及工具，\"/\"选择变量",
"agent.select_skill": "选择 Skill",
"agent.select_tool": "选择工具",
"agent.tools": "工具",
"append_application_reply_to_history_as_new_context": "将该应用回复内容拼接到历史记录中，作为新的上下文返回",
"application_call": "应用调用",
"assigned_reply": "指定回复",
......@@ -21,6 +25,7 @@
"auth_tmb_id_tip": "开启后，对外发布该应用时，还会根据用户是否有该知识库权限进行知识库过滤。\n若未开启，则直接按配置的知识库进行检索，不进行权限过滤。",
"auto_align": "自动对齐",
"can_not_loop": "该节点不支持循环嵌套",
"can_not_parallel": "该节点不支持在并行执行节点中使用",
"choose_another_application_to_call": "选择一个其他应用进行调用",
"classification_result": "分类结果",
"click_to_change_reference": "点击切换输入模式",
......@@ -41,9 +46,15 @@
"confirm_delete_field_tip": "确认删除该字段？",
"contains": "包含",
"content_to_retrieve": "需要检索的内容",
"content_to_search": "需要检索的内容",
"content_to_search": "检索词或检索语句",
"context_menu.add_comment": "添加注释",
"create_link_error": "创建链接异常",
"current_index": "当前下标",
"current_index_desc": "数组模式下当前元素的 0-based 下标。",
"current_item": "当前元素",
"current_item_desc": "数组模式下当前迭代处理的元素。",
"current_iteration": "当前循环次数",
"current_iteration_desc": "条件循环模式下的 1-based 迭代次数。",
"custom_feedback": "自定义反馈",
"custom_input": "自定义变量",
"dataset_quote_role": "角色",
......@@ -95,6 +106,8 @@
"intro_knowledge_base_search_merge": "可以将多个知识库搜索结果进行合并输出。使用 RRF 的合并方式进行最终排序输出。",
"intro_laf_function_call": "可以调用Laf账号下的云函数。",
"intro_loop": "输入一个数组，遍历数组并将每一个数组元素作为输入元素，执行工作流。",
"intro_loop_run": "执行重复性工作流，并由循环结果决定是否进行下一次操作。",
"intro_parallel_run": "输入一个数组，并行执行工作流中的每一个数组元素，最后将结果汇总输出。",
"intro_plugin_input": "可以配置插件需要哪些输入，利用这些输入来运行插件",
"intro_question_classification": "根据用户的历史记录和当前问题判断该次提问的类型。可以添加多组问题类型，下面是一个模板例子:\n类型1: 打招呼\n类型2: 关于商品“使用”问题\n类型3: 关于商品“购买”问题\n类型4: 其他问题",
"intro_question_optimization": "使用问题优化功能，可以提高知识库连续对话时搜索的精度。使用该功能后，会先利用 AI 根据上下文构建一个或多个新的检索词，这些检索词更利于进行知识库搜索。该模块已内置在知识库搜索模块中，如果您仅进行一次知识库搜索，可直接使用知识库内置的补全功能。",
......@@ -121,53 +134,28 @@
"less_than_or_equal_to": "小于等于",
"loop": "批量执行",
"loop_body": "循环体",
"loop_custom_outputs": "自定义输出",
"loop_custom_outputs_tip": "声明对外暴露的输出字段，每个字段引用子流程内某个节点的输出。",
"loop_end": "结束",
"loop_input_array": "数组",
"loop_result": "数组执行结果",
"loop_start": "开始",
"parallel_run": "并行执行",
"intro_parallel_run": "输入一个数组，并行执行工作流中的每一个数组元素，最后将结果汇总输出。",
"parallel_run_execution_logic": "执行逻辑",
"parallel_run_end_intro": "选择变量，作为并行执行的结果输出",
"parallel_run_max_concurrency": "最大并发数",
"parallel_run_max_concurrency_tip": "同时并行执行的最大任务数，范围 1～上限值（默认 5）。",
"parallel_run_max_retry_times": "单轮报错重试次数",
"parallel_run_max_retry_times_tip": "单个任务失败后的最大重试次数，范围 0～5（默认 3）。设为 0 表示失败不重试。",
"can_not_parallel": "该节点不支持在并行执行节点中使用",
"parallel_success_results": "成功结果",
"parallel_success_results_desc": "所有执行成功的任务输出，按输入顺序排列（仅包含成功项）。",
"parallel_full_results": "完整结果",
"parallel_full_results_desc": "与输入数组等长的结果数组，每项形如 {success, message, data}：成功时 success=true、message 为空、data 为输出值；失败时 success=false、message 为错误信息、data 为 null。",
"parallel_status": "完成状态",
"parallel_status_desc": "整体执行状态：success（全部成功）、partial_success（部分失败）、failed（全部失败）。",
"parallel_task": "任务 {{index}}",
"parallel_task_not_reach_end": "子工作流未到达结束节点",
"parallel_task_interactive_not_supported": "子工作流触发了交互式响应，并行执行节点不支持交互式节点",
"loop_run": "循环节点",
"intro_loop_run": "执行重复性工作流，并由循环结果决定是否进行下一次操作。",
"loop_run_mode": "循环类型",
"loop_run_mode_tip": "选择循环驱动方式：按数组元素依次执行，或持续执行直到命中「循环终止」。",
"loop_run_mode_array": "数组循环",
"loop_run_mode_array_desc": "按顺序处理列表内每一项",
"loop_run_mode_conditional": "条件循环",
"loop_run_mode_conditional_desc": "重复运行直至满足特定条件",
"loop_run_input_array": "数组",
"loop_custom_outputs": "自定义输出",
"loop_custom_outputs_tip": "声明对外暴露的输出字段，每个字段引用子流程内某个节点的输出。",
"loop_run_break": "循环终止",
"loop_run_break_tip": "该模块需要在循环节点内部使用。当该模块被执行时，将强制结束循环节点的运行。",
"loop_run_break_must_inside_loop_run": "循环终止节点只能放在循环执行节点内部",
"loop_run_break_tip": "该模块需要在循环节点内部使用。当该模块被执行时，将强制结束循环节点的运行。",
"loop_run_conditional_requires_break": "条件循环必须至少包含一个循环终止节点",
"loop_run_input_array": "数组",
"loop_run_input_not_array": "循环输入值不是数组",
"loop_run_iteration_failed": "循环执行失败",
"loop_run_max_iterations_exceeded": "循环达最大上限",
"loop_run_mode": "循环类型",
"loop_run_mode_array": "数组循环",
"loop_run_mode_array_desc": "按顺序处理列表内每一项",
"loop_run_mode_conditional": "条件循环",
"loop_run_mode_conditional_desc": "重复运行直至满足特定条件",
"loop_run_mode_tip": "选择循环驱动方式：按数组元素依次执行，或持续执行直到命中「循环终止」。",
"loop_run_start": "循环开始",
"current_index": "当前下标",
"current_index_desc": "数组模式下当前元素的 0-based 下标。",
"current_item": "当前元素",
"current_item_desc": "数组模式下当前迭代处理的元素。",
"current_iteration": "当前循环次数",
"current_iteration_desc": "条件循环模式下的 1-based 迭代次数。",
"loop_start": "开始",
"max_dialog_rounds": "最多携带多少轮对话记录",
"max_tokens": "最大 Tokens",
"mouse_priority": "鼠标优先\n- 左键按下后可拖动画布\n- 按住 Shift 并拖动可框选多个节点",
......@@ -179,6 +167,23 @@
"optional_value_type": "可选的数据类型",
"optional_value_type_tip": "可以指定 1 个或多个数据类型，用户在动态添加字段时，仅可选择配置的类型",
"pan_priority": "触摸板优先\n- 单指拖拽框选节点\n- 双指滑动移动画布\n- 三指可快速移动节点",
"parallel_full_results": "完整结果",
"parallel_full_results_desc": "与输入数组等长的结果数组，每项形如 {success, message, data}：成功时 success=true、message 为空、data 为输出值；失败时 success=false、message 为错误信息、data 为 null。",
"parallel_run": "并行执行",
"parallel_run_end_intro": "选择变量，作为并行执行的结果输出",
"parallel_run_execution_logic": "执行逻辑",
"parallel_run_max_concurrency": "最大并发数",
"parallel_run_max_concurrency_tip": "同时并行执行的最大任务数，范围 1～上限值（默认 5）。",
"parallel_run_max_retry_times": "单轮报错重试次数",
"parallel_run_max_retry_times_tip": "单个任务失败后的最大重试次数，范围 0～5（默认 3）。设为 0 表示失败不重试。",
"parallel_status": "完成状态",
"parallel_status_desc": "整体执行状态：success（全部成功）、partial_success（部分失败）、failed（全部失败）。",
"parallel_success_results": "成功结果",
"parallel_success_results_desc": "所有执行成功的任务输出，按输入顺序排列（仅包含成功项）。",
"parallel_task": "任务 {{index}}",
"parallel_task_interactive_not_supported": "子工作流触发了交互式响应，并行执行节点不支持交互式节点",
"parallel_task_not_reach_end": "子工作流未到达结束节点",
"params_setting": "参数设置",
"pass_returned_object_as_output_to_next_nodes": "将代码中 return 的对象作为输出，传递给后续的节点。变量名需要对应 return 的 key",
"please_enter_node_name": "请输入节点名称",
"plugin.Instruction_Tip": "可以配置一段说明，以解释该插件的用途。每次使用插件前，会显示该段说明。支持标准 Markdown 语法。",
......@@ -205,6 +210,7 @@
"response.Error": "错误信息",
"response.Read file result": "文档解析结果预览",
"response.read files": "解析的文档",
"search_query": "检索内容",
"select_an_application": "选择一个应用",
"select_another_application_to_call": "可以选择一个其他应用进行调用",
"select_default_option": "选择默认值",
......@@ -215,10 +221,6 @@
"template.agent_intro": "由 AI 自主决定工具调用。",
"template.agent_module": "Agent",
"template.agent_module_intro": "关联所需工具及 Skill，可实现 AI 自主调用与规划",
"agent.tools": "工具",
"agent.prompt_skill_tip": "@选择 Skill 及工具，\"/\"选择变量",
"agent.select_skill": "选择 Skill",
"agent.select_tool": "选择工具",
"template.ai_chat": "AI 对话",
"template.ai_chat_intro": "AI 大模型对话",
"template.dataset_search": "知识库搜索",
......@@ -271,6 +273,5 @@
"workflow.My edit": "我的编辑",
"workflow.Switch_success": "切换成功",
"workflow.Team cloud": "团队云端",
"workflow.exit_tips": "您的更改尚未保存，「直接退出」将不会保存您的编辑记录。",
"params_setting": "参数设置"
"workflow.exit_tips": "您的更改尚未保存，「直接退出」将不会保存您的编辑记录。"
}
packages/web/i18n/zh-Hant/workflow.json
......@@ -14,6 +14,10 @@
"Variable_name": "變數名稱",
"add_new_input": "新增輸入",
"add_new_output": "新增輸出",
"agent.prompt_skill_tip": "@選擇 Skill 及工具，\"/\"選擇變數",
"agent.select_skill": "選擇 Skill",
"agent.select_tool": "選擇工具",
"agent.tools": "工具",
"append_application_reply_to_history_as_new_context": "將應用程式的回覆附加到歷史紀錄中，作為新的脈絡",
"application_call": "應用程式呼叫",
"assigned_reply": "指定回覆",
......@@ -21,6 +25,7 @@
"auth_tmb_id_tip": "開啟後，對外發布應用程式時，也會根據使用者是否有該知識庫權限進行知識庫過濾。\n\n若未開啟，則直接按設定的知識庫進行檢索，不進行權限過濾。",
"auto_align": "自動對齊",
"can_not_loop": "這個節點不能迴圈。",
"can_not_parallel": "該節點不支援在並行執行節點中使用",
"choose_another_application_to_call": "選擇另一個應用程式來呼叫",
"classification_result": "分類結果",
"click_to_change_reference": "點擊切換輸入模式",
......@@ -41,9 +46,15 @@
"confirm_delete_field_tip": "確定要刪除這個欄位嗎？",
"contains": "包含",
"content_to_retrieve": "要擷取的內容",
"content_to_search": "要搜尋的內容",
"content_to_search": "檢索字或檢索語句",
"context_menu.add_comment": "新增註解",
"create_link_error": "建立連結發生錯誤",
"current_index": "目前索引",
"current_index_desc": "陣列模式下目前元素的 0-based 索引。",
"current_item": "目前元素",
"current_item_desc": "陣列模式下目前迭代處理的元素。",
"current_iteration": "目前迴圈次數",
"current_iteration_desc": "條件迴圈模式下的 1-based 迭代次數。",
"custom_feedback": "自訂回饋",
"custom_input": "自定義變量",
"dataset_quote_role": "角色",
......@@ -95,6 +106,8 @@
"intro_knowledge_base_search_merge": "可以合併多個知識庫搜尋結果並輸出。使用 RRF 合併方法進行最終排序輸出。",
"intro_laf_function_call": "可以呼叫 Laf 帳號下的雲端函式。",
"intro_loop": "輸入一個陣列，遍歷陣列並將每個陣列元素作為輸入元素，執行工作流程。",
"intro_loop_run": "執行重複性工作流，由迴圈結果決定是否進行下一次操作。",
"intro_parallel_run": "輸入一個陣列，並行執行工作流中的每一個陣列元素，最後將結果匯總輸出。",
"intro_plugin_input": "可以設定外掛程式需要的輸入，並利用這些輸入來執行外掛程式",
"intro_question_classification": "根據使用者的歷史紀錄和目前問題判斷這次提問的類型。可以新增多個問題類型，以下是一個範例：\n類型 1：打招呼\n類型 2：關於產品「使用方式」的問題\n類型 3：關於產品「購買」的問題\n類型 4：其他問題",
"intro_question_optimization": "使用問題最佳化功能，可以提升知識庫連續對話時的搜尋精準度。使用這個功能後，會先利用 AI 根據脈絡建構一個或多個新的檢索詞彙，這些詞彙更有利於知識庫搜尋。這個模組已內建於知識庫搜尋模組中，如果您只進行一次知識庫搜尋，可以直接使用知識庫內建的自動完成功能。",
......@@ -121,53 +134,28 @@
"less_than_or_equal_to": "小於或等於",
"loop": "大量執行",
"loop_body": "迴圈主體",
"loop_custom_outputs": "自訂輸出",
"loop_custom_outputs_tip": "宣告對外曝露的輸出欄位，每個欄位引用子流程內某個節點的輸出。",
"loop_end": "結束",
"loop_input_array": "陣列",
"loop_result": "陣列執行結果",
"loop_start": "開始",
"parallel_run": "並行執行",
"intro_parallel_run": "輸入一個陣列，並行執行工作流中的每一個陣列元素，最後將結果匯總輸出。",
"parallel_run_execution_logic": "執行邏輯",
"parallel_run_end_intro": "選擇變數，作為並行執行的結果輸出",
"parallel_run_max_concurrency": "最大並發數",
"parallel_run_max_concurrency_tip": "同時並行執行的最大任務數，範圍 1～上限值（預設 5）。",
"parallel_run_max_retry_times": "單輪報錯重試次數",
"parallel_run_max_retry_times_tip": "單個任務失敗後的最大重試次數，範圍 0～5（預設 3）。設為 0 表示失敗不重試。",
"can_not_parallel": "該節點不支援在並行執行節點中使用",
"parallel_success_results": "成功結果",
"parallel_success_results_desc": "所有執行成功的任務輸出，按輸入順序排列（僅包含成功項）。",
"parallel_full_results": "完整結果",
"parallel_full_results_desc": "與輸入陣列等長的結果陣列，每項形如 {success, message, data}：成功時 success=true、message 為空、data 為輸出值；失敗時 success=false、message 為錯誤訊息、data 為 null。",
"parallel_status": "完成狀態",
"parallel_status_desc": "整體執行狀態：success（全部成功）、partial_success（部分失敗）、failed（全部失敗）。",
"parallel_task": "任務 {{index}}",
"parallel_task_not_reach_end": "子工作流未到達結束節點",
"parallel_task_interactive_not_supported": "子工作流觸發了互動式回應，並行執行節點不支援互動式節點",
"loop_run": "迴圈節點",
"intro_loop_run": "執行重複性工作流，由迴圈結果決定是否進行下一次操作。",
"loop_run_mode": "迴圈類型",
"loop_run_mode_tip": "選擇迴圈驅動方式：依陣列元素逐一執行，或持續執行直到命中「迴圈終止」。",
"loop_run_mode_array": "陣列迴圈",
"loop_run_mode_array_desc": "依序處理清單內每一項",
"loop_run_mode_conditional": "條件迴圈",
"loop_run_mode_conditional_desc": "重複執行直到滿足特定條件",
"loop_run_input_array": "陣列",
"loop_custom_outputs": "自訂輸出",
"loop_custom_outputs_tip": "宣告對外曝露的輸出欄位，每個欄位引用子流程內某個節點的輸出。",
"loop_run_break": "迴圈終止",
"loop_run_break_tip": "此模組需要在迴圈節點內部使用。當此模組被執行時，將強制結束迴圈節點的執行。",
"loop_run_break_must_inside_loop_run": "迴圈終止節點只能放在迴圈執行節點內部",
"loop_run_break_tip": "此模組需要在迴圈節點內部使用。當此模組被執行時，將強制結束迴圈節點的執行。",
"loop_run_conditional_requires_break": "條件迴圈必須至少包含一個迴圈終止節點",
"loop_run_input_array": "陣列",
"loop_run_input_not_array": "迴圈輸入值不是陣列",
"loop_run_iteration_failed": "迴圈執行失敗",
"loop_run_max_iterations_exceeded": "迴圈達最大上限",
"loop_run_mode": "迴圈類型",
"loop_run_mode_array": "陣列迴圈",
"loop_run_mode_array_desc": "依序處理清單內每一項",
"loop_run_mode_conditional": "條件迴圈",
"loop_run_mode_conditional_desc": "重複執行直到滿足特定條件",
"loop_run_mode_tip": "選擇迴圈驅動方式：依陣列元素逐一執行，或持續執行直到命中「迴圈終止」。",
"loop_run_start": "迴圈開始",
"current_index": "目前索引",
"current_index_desc": "陣列模式下目前元素的 0-based 索引。",
"current_item": "目前元素",
"current_item_desc": "陣列模式下目前迭代處理的元素。",
"current_iteration": "目前迴圈次數",
"current_iteration_desc": "條件迴圈模式下的 1-based 迭代次數。",
"loop_start": "開始",
"max_dialog_rounds": "最多攜帶幾輪對話紀錄",
"max_tokens": "最大 Token 數",
"mouse_priority": "滑鼠優先\n- 按下左鍵拖曳畫布\n- 按住 Shift 並拖曳可框選多個節點",
......@@ -179,6 +167,23 @@
"optional_value_type": "可選的資料類型",
"optional_value_type_tip": "可以指定一或多個資料類型，使用者在動態新增欄位時，只能選擇已設定的類型",
"pan_priority": "觸摸板優先\n- 單指拖拽框選節點\n- 雙指滑動移動畫布\n- 三指可快速移動節點",
"parallel_full_results": "完整結果",
"parallel_full_results_desc": "與輸入陣列等長的結果陣列，每項形如 {success, message, data}：成功時 success=true、message 為空、data 為輸出值；失敗時 success=false、message 為錯誤訊息、data 為 null。",
"parallel_run": "並行執行",
"parallel_run_end_intro": "選擇變數，作為並行執行的結果輸出",
"parallel_run_execution_logic": "執行邏輯",
"parallel_run_max_concurrency": "最大並發數",
"parallel_run_max_concurrency_tip": "同時並行執行的最大任務數，範圍 1～上限值（預設 5）。",
"parallel_run_max_retry_times": "單輪報錯重試次數",
"parallel_run_max_retry_times_tip": "單個任務失敗後的最大重試次數，範圍 0～5（預設 3）。設為 0 表示失敗不重試。",
"parallel_status": "完成狀態",
"parallel_status_desc": "整體執行狀態：success（全部成功）、partial_success（部分失敗）、failed（全部失敗）。",
"parallel_success_results": "成功結果",
"parallel_success_results_desc": "所有執行成功的任務輸出，按輸入順序排列（僅包含成功項）。",
"parallel_task": "任務 {{index}}",
"parallel_task_interactive_not_supported": "子工作流觸發了互動式回應，並行執行節點不支援互動式節點",
"parallel_task_not_reach_end": "子工作流未到達結束節點",
"params_setting": "參數設定",
"pass_returned_object_as_output_to_next_nodes": "將程式碼中 return 的物件作為輸出，傳遞給後續的節點。變數名稱需要對應 return 的鍵值",
"please_enter_node_name": "請輸入節點名稱",
"plugin.Instruction_Tip": "您可以設定一段說明來解釋這個外掛程式的用途。每次使用外掛程式前，都會顯示這段說明。支援標準 Markdown 語法。",
......@@ -205,6 +210,7 @@
"response.Error": "錯誤訊息",
"response.Read file result": "檔案解析結果預覽",
"response.read files": "解析的檔案",
"search_query": "檢索內容",
"select_an_application": "選擇一個應用程式",
"select_another_application_to_call": "可以選擇另一個應用程式來呼叫",
"select_default_option": "選擇默認值",
......@@ -215,10 +221,6 @@
"template.agent_intro": "透過 AI 模型自動選擇一或多個功能區塊進行呼叫，也可以呼叫外掛程式。",
"template.agent_module": "Agent",
"template.agent_module_intro": "關聯所需工具及 Skill，可實現 AI 自主呼叫與規劃",
"agent.tools": "工具",
"agent.prompt_skill_tip": "@選擇 Skill 及工具，\"/\"選擇變數",
"agent.select_skill": "選擇 Skill",
"agent.select_tool": "選擇工具",
"template.ai_chat": "AI 對話",
"template.ai_chat_intro": "AI 大型語言模型對話",
"template.dataset_search": "知識庫搜尋",
......@@ -271,6 +273,5 @@
"workflow.My edit": "我的編輯",
"workflow.Switch_success": "切換成功",
"workflow.Team cloud": "團隊雲端",
"workflow.exit_tips": "您的變更尚未儲存，「直接結束」將不會儲存您的編輯紀錄。",
"params_setting": "參數設定"
"workflow.exit_tips": "您的變更尚未儲存，「直接結束」將不會儲存您的編輯紀錄。"
}
pro @ fa441cde
Subproject commit cde23cd5f0cda7d89aa8b9ea66b51fa2f0d4fa2d
Subproject commit fa441cde179aac9113f67941216c8d990c5ba681
projects/app/src/components/core/chat/components/WholeResponseModal/ImageQuery.tsx
import ImagePreviewToken from '@/components/core/dataset/ImagePreviewToken';
import type { ImagePreviewTokenItemType } from '@/components/core/dataset/ImagePreviewToken';
import { Box } from '@chakra-ui/react';
import type { ChatHistoryItemResType } from '@fastgpt/global/core/chat/type';
import React from 'react';
const httpUrlReg = /^https?:\/\//i;
/**
* nodeResponse 里为了兼容历史展示，把文本 query 和图片 URL 都放在 datasetQueries。
* UI 侧只负责展示拆分：普通 http(s) URL 作为图片 token，其余内容按原文本换行展示。
*/
const splitDatasetQueries = (datasetQueries: string[]) => {
const textQueries: string[] = [];
const queryImages: ImagePreviewTokenItemType[] = [];
for (const query of datasetQueries) {
if (!query) continue;
if (httpUrlReg.test(query)) {
queryImages.push({ url: query });
} else {
textQueries.push(query);
}
}
return {
query: textQueries.join('\n'),
queryImages
};
};
const ImageQuery = ({
query,
queryImages,
datasetQueries,
datasetId
}: {
query?: string;
queryImages: NonNullable<ChatHistoryItemResType['queryImages']>;
datasetQueries: string[];
datasetId?: string;
}) => {
const { query, queryImages } = splitDatasetQueries(datasetQueries);
return (
<Box
border={'1px solid'}
......
projects/app/src/components/core/chat/components/WholeResponseModal/ResponseRows.tsx
......@@ -167,20 +167,7 @@ export const AiChatRows = ({
/>
<Row label={t('chat:step_query')} value={activeModule.stepQuery} />
{activeModule.queryImages && activeModule.queryImages.length > 0 ? (
<Row
label={t('common:core.chat.response.module query')}
rawDom={
<ImageQuery
query={activeModule.query}
queryImages={activeModule.queryImages}
datasetId={queryPreviewDatasetId}
/>
}
/>
) : (
<Row label={t('common:core.chat.response.module query')} value={activeModule.query} />
)}
<Row label={t('common:core.chat.response.module query')} value={activeModule.query} />
<Row
label={t('common:core.chat.response.context total length')}
value={activeModule.contextTotalLen}
......@@ -251,6 +238,17 @@ export const DatasetSearchRows = ({
label={t('common:core.chat.response.module similarity')}
value={activeModule.similarity}
/>
{activeModule.datasetQueries && activeModule.datasetQueries.length > 0 && (
<Row
label={t('common:core.chat.response.module query')}
rawDom={
<ImageQuery
datasetQueries={activeModule.datasetQueries}
datasetId={activeModule.quoteList?.[0]?.datasetId}
/>
}
/>
)}
<Row label={t('common:core.chat.response.module limit')} value={activeModule.limit} />
<Row label={t('chat:response_embedding_model')} value={activeModule.embeddingModel} />
<Row
......
projects/app/src/components/core/dataset/ImagePreviewToken.tsx
import React, { useEffect, useState } from 'react';
import { Box, Flex, Portal, type BoxProps, type FlexProps } from '@chakra-ui/react';
import React, { useCallback, useEffect, useState } from 'react';
import {
Box,
CircularProgress,
Flex,
Portal,
type BoxProps,
type FlexProps
} from '@chakra-ui/react';
import { isDatasetFileObjectKey } from '@fastgpt/global/core/dataset/utils';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { MyPhotoSlider } from '@fastgpt/web/components/common/Image/PhotoView';
import { useSafeTranslation } from '@fastgpt/web/hooks/useSafeTranslation';
import { postGetSearchTestImagePreviewUrls } from '@/web/core/dataset/api/file';
......@@ -12,52 +20,115 @@ export type ImagePreviewTokenItemType = {
name?: string;
};
type PreviewCache =
| {
status: 'ready';
url: string;
}
| {
status: 'expired';
};
const getDirectPreviewUrl = (image: ImagePreviewTokenItemType) => {
const url = image.previewUrl || image.url || '';
return url && !isDatasetFileObjectKey(url) ? url : '';
};
const getImageCacheKey = (image: ImagePreviewTokenItemType, index: number) =>
image.key || image.url || image.previewUrl || `${index}`;
const PREVIEW_URL_TIMEOUT_MS = 5000;
const withTimeout = async <T,>(promise: Promise<T>, timeoutMs: number) => {
let timer: ReturnType<typeof setTimeout>;
return Promise.race([
promise,
new Promise<never>((_, reject) => {
timer = setTimeout(() => reject(new Error('Search test image preview timeout')), timeoutMs);
})
]).finally(() => {
clearTimeout(timer);
});
};
const ImagePreview = React.memo(function ImagePreview({
image,
datasetId
datasetId,
cachedPreviewUrl,
onPreviewUrlChange,
onPreviewExpired
}: {
image: ImagePreviewTokenItemType;
datasetId?: string;
cachedPreviewUrl?: string;
onPreviewUrlChange?: (previewUrl: string) => void;
onPreviewExpired?: () => void;
}) {
const { t } = useSafeTranslation();
const [previewUrl, setPreviewUrl] = useState(() => getDirectPreviewUrl(image));
const [previewUrl, setPreviewUrl] = useState(
() => getDirectPreviewUrl(image) || cachedPreviewUrl
);
const [loadFailed, setLoadFailed] = useState(false);
const [hasRefreshed, setHasRefreshed] = useState(false);
const [isRefreshing, setIsRefreshing] = useState(false);
useEffect(() => {
setPreviewUrl(getDirectPreviewUrl(image));
setPreviewUrl(getDirectPreviewUrl(image) || cachedPreviewUrl || '');
setLoadFailed(false);
setHasRefreshed(false);
}, [image]);
setIsRefreshing(false);
}, [cachedPreviewUrl, image]);
useEffect(() => {
if (!image.key || !datasetId || hasRefreshed || (previewUrl && !loadFailed)) return;
let canceled = false;
setHasRefreshed(true);
setIsRefreshing(true);
postGetSearchTestImagePreviewUrls({
datasetId,
keys: [image.key]
})
withTimeout(
postGetSearchTestImagePreviewUrls({
datasetId,
keys: [image.key]
}),
PREVIEW_URL_TIMEOUT_MS
)
.then((res) => {
const nextPreviewUrl = res.find((item) => item.key === image.key)?.previewUrl;
if (!canceled && nextPreviewUrl) {
setPreviewUrl(nextPreviewUrl);
setLoadFailed(false);
if (!canceled) {
if (nextPreviewUrl) {
setPreviewUrl(nextPreviewUrl);
onPreviewUrlChange?.(nextPreviewUrl);
setLoadFailed(false);
} else {
onPreviewExpired?.();
}
}
})
.catch(() => {});
.catch(() => {
if (!canceled) {
onPreviewExpired?.();
}
})
.finally(() => {
if (!canceled) {
setIsRefreshing(false);
}
});
return () => {
canceled = true;
};
}, [datasetId, hasRefreshed, image.key, loadFailed, previewUrl]);
}, [
datasetId,
hasRefreshed,
image.key,
loadFailed,
onPreviewExpired,
onPreviewUrlChange,
previewUrl
]);
if (previewUrl && !loadFailed) {
return (
......@@ -69,11 +140,33 @@ const ImagePreview = React.memo(function ImagePreview({
h={'80px'}
objectFit={'cover'}
borderRadius={'sm'}
onError={() => setLoadFailed(true)}
onError={() => {
setLoadFailed(true);
if (!image.key || !datasetId || hasRefreshed) {
onPreviewExpired?.();
}
}}
/>
);
}
if (image.key && datasetId && (!hasRefreshed || isRefreshing)) {
return (
<Flex
w={'80px'}
h={'80px'}
alignItems={'center'}
justifyContent={'center'}
bg={'myGray.50'}
border={'1px dashed'}
borderColor={'myGray.300'}
borderRadius={'sm'}
>
<CircularProgress isIndeterminate size={'24px'} color={'primary.600'} />
</Flex>
);
}
return (
<Flex
w={'80px'}
......@@ -106,7 +199,7 @@ const defaultTokenStyles: BoxProps = {
borderRadius: 'md',
bg: 'white',
color: 'myGray.700',
cursor: 'default',
cursor: 'pointer',
lineHeight: '16px',
verticalAlign: 'baseline'
};
......@@ -126,35 +219,131 @@ const ImagePreviewToken = React.memo(function ImagePreviewToken({
const [hoveredImage, setHoveredImage] = useState<
| {
image: ImagePreviewTokenItemType;
cacheKey: string;
top: number;
left: number;
}
| undefined
>();
const [previewCacheMap, setPreviewCacheMap] = useState<Record<string, PreviewCache>>({});
const [viewerImage, setViewerImage] = useState<
| {
cacheKey: string;
src: string;
}
| undefined
>();
const updatePreviewUrl = useCallback((cacheKey: string, previewUrl: string) => {
setPreviewCacheMap((state) => ({
...state,
[cacheKey]: {
status: 'ready',
url: previewUrl
}
}));
}, []);
const markPreviewExpired = useCallback((cacheKey: string) => {
setPreviewCacheMap((state) => ({
...state,
[cacheKey]: {
status: 'expired'
}
}));
}, []);
const resolvePreviewUrl = useCallback(
async (image: ImagePreviewTokenItemType, index: number) => {
const cacheKey = getImageCacheKey(image, index);
const previewCache = previewCacheMap[cacheKey];
if (previewCache?.status === 'expired') return '';
const directPreviewUrl = getDirectPreviewUrl(image);
if (directPreviewUrl) return directPreviewUrl;
if (previewCache?.status === 'ready') return previewCache.url;
if (!image.key || !datasetId) return '';
const previewUrl = await withTimeout(
postGetSearchTestImagePreviewUrls({
datasetId,
keys: [image.key]
}),
PREVIEW_URL_TIMEOUT_MS
)
.then((previewUrls) => previewUrls.find((item) => item.key === image.key)?.previewUrl || '')
.catch(() => '');
if (previewUrl) {
updatePreviewUrl(cacheKey, previewUrl);
} else {
markPreviewExpired(cacheKey);
}
return previewUrl;
},
[datasetId, markPreviewExpired, previewCacheMap, updatePreviewUrl]
);
if (images.length === 0) return null;
const hoveredPreviewCache = hoveredImage ? previewCacheMap[hoveredImage.cacheKey] : undefined;
const hoveredPreviewUrl =
hoveredPreviewCache?.status === 'ready' ? hoveredPreviewCache.url : undefined;
return (
<>
<Flex flexWrap={'wrap'} gap={2} {...containerProps}>
{images.map((image, index) => (
<Box
key={`${image.key || image.url || image.previewUrl || index}`}
{...defaultTokenStyles}
{...tokenProps}
onMouseEnter={(e) => {
const rect = e.currentTarget.getBoundingClientRect();
setHoveredImage({
image,
top: rect.bottom + 8,
left: rect.left
});
}}
onMouseLeave={() => setHoveredImage(undefined)}
>
{t('common:core.dataset.test.image_token')}
</Box>
))}
{images.map((image, index) => {
const cacheKey = getImageCacheKey(image, index);
const isPreviewExpired = previewCacheMap[cacheKey]?.status === 'expired';
return (
<Box
key={cacheKey}
{...defaultTokenStyles}
{...tokenProps}
cursor={isPreviewExpired ? 'default' : 'pointer'}
role={isPreviewExpired ? undefined : 'button'}
tabIndex={isPreviewExpired ? undefined : 0}
aria-label={
isPreviewExpired
? t('common:core.dataset.test.image_expired')
: t('common:Click_to_expand')
}
onMouseEnter={(e) => {
const rect = e.currentTarget.getBoundingClientRect();
setHoveredImage({
image,
cacheKey,
top: rect.bottom + 8,
left: rect.left
});
}}
onMouseLeave={() => setHoveredImage(undefined)}
onClick={async (e) => {
e.stopPropagation();
if (isPreviewExpired) return;
const previewUrl = await resolvePreviewUrl(image, index);
if (!previewUrl) return;
setViewerImage({
cacheKey,
src: previewUrl
});
}}
onKeyDown={(e) => {
if (isPreviewExpired || (e.key !== 'Enter' && e.key !== ' ')) return;
e.preventDefault();
e.currentTarget.click();
}}
>
{t('common:core.dataset.test.image_token')}
</Box>
);
})}
</Flex>
{!!hoveredImage && (
......@@ -172,10 +361,25 @@ const ImagePreviewToken = React.memo(function ImagePreviewToken({
boxShadow={'2'}
pointerEvents={'none'}
>
<ImagePreview image={hoveredImage.image} datasetId={datasetId} />
<ImagePreview
image={hoveredImage.image}
datasetId={datasetId}
cachedPreviewUrl={hoveredPreviewUrl}
onPreviewUrlChange={(previewUrl) =>
updatePreviewUrl(hoveredImage.cacheKey, previewUrl)
}
onPreviewExpired={() => markPreviewExpired(hoveredImage.cacheKey)}
/>
</Flex>
</Portal>
)}
<MyPhotoSlider
src={viewerImage?.src}
visible={!!viewerImage}
onClose={() => setViewerImage(undefined)}
imageKey={viewerImage?.cacheKey}
/>
</>
);
});
......
projects/app/src/pageComponents/app/detail/Edit/SimpleApp/utils.ts
......@@ -385,7 +385,7 @@ export function form2AppWorkflow(
return {
nodeId: datasetNodeId,
name: t(DatasetSearchModule.name),
intro: t('app:dataset_search_tool_description'),
intro: DatasetSearchModule.intro,
avatar: DatasetSearchModule.avatar,
flowNodeType: DatasetSearchModule.flowNodeType,
showStatus: true,
......@@ -476,6 +476,8 @@ export function form2AppWorkflow(
},
{
...Input_Template_UserChatInput,
key: NodeInputKeyEnum.datasetSearchInput,
valueType: WorkflowIOValueTypeEnum.arrayString,
toolDescription: i18nT('workflow:content_to_search'),
value: question
}
......@@ -503,7 +505,10 @@ export function form2AppWorkflow(
return {
nodes: [
aiChatTemplate(formData),
datasetNodeTemplate(formData, [workflowStartNodeId, 'userChatInput'])
datasetNodeTemplate(formData, [
[workflowStartNodeId, NodeOutputKeyEnum.userChatInput],
[workflowStartNodeId, NodeOutputKeyEnum.userFiles]
])
],
edges: [
{
......
projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/components/NodeTemplates/list.tsx
......@@ -264,6 +264,7 @@ const NodeTemplateList = ({
const defaultValueMap: Record<string, any> = {
[NodeInputKeyEnum.userChatInput]: undefined,
[NodeInputKeyEnum.datasetSearchInput]: undefined,
[NodeInputKeyEnum.fileUrlList]: undefined
};
......@@ -276,6 +277,10 @@ const NodeTemplateList = ({
defaultValueMap[NodeInputKeyEnum.fileUrlList] = [
[node.nodeId, NodeOutputKeyEnum.userFiles]
];
defaultValueMap[NodeInputKeyEnum.datasetSearchInput] = [
[node.nodeId, NodeOutputKeyEnum.userChatInput],
[node.nodeId, NodeOutputKeyEnum.userFiles]
];
}
});
......@@ -413,7 +418,7 @@ const NodeTemplateList = ({
[
computedNewNodeName,
getNodeById,
handleParams?.nodeId,
handleParams,
getNodeList,
getIntersectingNodes,
onAddNode,
......@@ -488,7 +493,7 @@ const NodeTemplateList = ({
return data.filter(({ list }) => list.length > 0);
}, [templateType, templates, t, i18n.language]);
const NodeListRender = useMemoizedFn(({ list = [] }: { list: NodeTemplateListType }) => {
const renderNodeList = useMemoizedFn((list: NodeTemplateListType = []) => {
return (
<>
{list.map((item) => {
......@@ -559,14 +564,12 @@ const NodeTemplateList = ({
{t(label as any)}
<AccordionIcon />
</AccordionButton>
<AccordionPanel py={0}>
<NodeListRender list={list} />
</AccordionPanel>
<AccordionPanel py={0}>{renderNodeList(list)}</AccordionPanel>
</AccordionItem>
))}
</>
) : (
<NodeListRender list={formatTemplatesArrayData?.[0]?.list} />
<>{renderNodeList(formatTemplatesArrayData?.[0]?.list)}</>
)}
</Accordion>
</Box>
......
projects/app/src/pageComponents/dataset/detail/Test/components/TestHistories.tsx
......@@ -52,10 +52,10 @@ const TestHistories = ({
borderColor: 'primary.300',
boxShadow: '1',
'& .delete': {
display: 'block'
visibility: 'visible'
},
'& .time': {
display: 'none'
visibility: 'hidden'
}
}}
cursor={'pointer'}
......@@ -97,7 +97,7 @@ const TestHistories = ({
{t(formatTimeToChatTime(item.time) as any).replace('#', ':')}
</Box>
<MyTooltip label={t('common:core.dataset.test.delete test history')}>
<Box className="delete" display={'none'} w={'0.8rem'} h={'0.8rem'} ml={1}>
<Box className="delete" visibility={'hidden'} w={'0.8rem'} h={'0.8rem'} ml={1}>
<MyIcon
name={'delete'}
w={'0.8rem'}
......
projects/app/src/pageComponents/dataset/detail/Test/constants.ts
import { imageFileType } from '@fastgpt/global/common/file/constants';
// Keep the disabled upload path in place so the search test flow can enable images by flag only.
export const SEARCH_TEST_IMAGE_UPLOAD_ENABLED = false;
export const SEARCH_TEST_IMAGE_UPLOAD_ENABLED = true;
export const MAX_SEARCH_TEST_IMAGE_COUNT = 10;
// Derive accepted extensions from the shared upload accept string to avoid frontend/backend drift.
......
projects/app/src/pageComponents/dataset/detail/Test/hooks/useDatasetSearchTest.ts
......@@ -79,9 +79,7 @@ export const useDatasetSearchTest = ({
text: getValues('inputText').trim(),
time: new Date(),
results: res.list,
// Histories only need stable image keys; preview urls are regenerated by ImagePreviewToken.
queryImageRefs:
queryImageRefs.length > 0 ? queryImageRefs.map(({ key }) => ({ key })) : undefined,
queryImageRefs: queryImageRefs.length > 0 ? queryImageRefs : undefined,
duration: res.duration,
searchMode: res.searchMode,
usingReRank: res.usingReRank,
......
projects/app/src/pages/api/core/ai/model/test.ts
......@@ -11,7 +11,7 @@ import {
} from '@fastgpt/global/core/ai/model.schema';
import { getAIApi } from '@fastgpt/service/core/ai/config';
import { getLogger, LogCategories } from '@fastgpt/service/common/logger';
import { getVectorsByText } from '@fastgpt/service/core/ai/embedding';
import { getVectors } from '@fastgpt/service/core/ai/embedding';
import { reRankRecall } from '@fastgpt/service/core/ai/rerank';
import { aiTranscriptions } from '@fastgpt/service/core/ai/audio/transcriptions';
import { isProduction } from '@fastgpt/global/common/system/constants';
......@@ -87,9 +87,14 @@ const testEmbeddingModel = async (
model: EmbeddingModelItemType,
headers: Record<string, string>
) => {
return getVectorsByText({
input: 'Hi',
return getVectors({
model,
inputs: [
{
type: 'text',
input: 'Hi'
}
],
headers
});
};
......
projects/app/src/pages/api/core/dataset/searchTest.ts
import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
import { pushDatasetTestUsage } from '@/service/support/wallet/usage/push';
import {
deepRagSearch,
defaultSearchDatasetData
} from '@fastgpt/service/core/dataset/search/controller';
import { deepRagSearch, defaultSearchDatasetData } from '@fastgpt/service/core/dataset/search';
import { updateApiKeyUsage } from '@fastgpt/service/support/openapi/tools';
import { UsageSourceEnum } from '@fastgpt/global/support/wallet/usage/constants';
import { checkTeamAIPoints } from '@fastgpt/service/support/permission/teamLimit';
......@@ -15,6 +12,8 @@ import { getRerankModel } from '@fastgpt/service/core/ai/model';
import { addAuditLog } from '@fastgpt/service/support/user/audit/util';
import { AuditEventEnum } from '@fastgpt/global/support/user/audit/constants';
import { getI18nDatasetType } from '@fastgpt/service/support/user/audit/util';
import { isS3ObjectKey } from '@fastgpt/service/common/s3/utils';
import { getS3DatasetSource } from '@fastgpt/service/common/s3/sources/dataset';
import {
SearchDatasetTestBodySchema,
SearchDatasetTestResponseSchema,
......@@ -22,12 +21,13 @@ import {
type SearchDatasetTestResponse
} from '@fastgpt/global/openapi/core/dataset/api';
async function handler(
export async function handler(
req: ApiRequestProps<SearchDatasetTestBody>
): Promise<SearchDatasetTestResponse> {
const {
datasetId,
text,
queryImageUrls,
limit = 5000,
similarity,
searchMode,
......@@ -50,7 +50,7 @@ async function handler(
const start = Date.now();
// auth dataset role
const { dataset, teamId, tmbId, userId, apikey } = await authDataset({
const { dataset, teamId, tmbId, apikey } = await authDataset({
req,
authToken: true,
authApiKey: true,
......@@ -60,14 +60,37 @@ async function handler(
// auth balance
await checkTeamAIPoints(teamId);
// Search-test images must be temp objects created by this team. Client-supplied keys are not
// proof of ownership, so reject dataset/chat/foreign-team keys before any S3 read happens.
const validQueryImageKeys = queryImageUrls.filter(
(key) => isS3ObjectKey(key, 'temp') && key.startsWith(`temp/${teamId}/`)
);
if (validQueryImageKeys.length !== queryImageUrls.length) {
return Promise.reject('Invalid query image key');
}
// 搜索主链路只接收模型可读图片 URL；temp key 的鉴权和临时 URL 生成固定在入口层完成。
const validQueryImageUrls = await Promise.all(
validQueryImageKeys.map(async (key) => {
const { url } = await getS3DatasetSource().createExternalUrl({
key,
expiredHours: 1
});
return url;
})
);
const rerankModelData = getRerankModel(rerankModel);
const searchData = {
histories: [],
teamId,
reRankQuery: text,
queries: [text],
textQueries: text ? [text] : [],
imageQueries: validQueryImageUrls,
model: dataset.vectorModel,
vlmModel: dataset.vlmModel,
limit: Math.min(limit, 20000),
similarity,
datasetIds: [datasetId],
......@@ -83,9 +106,9 @@ async function handler(
reRankInputTokens,
usingReRank: searchUsingReRank,
queryExtensionResult,
deepSearchResult,
imageCaptionResult,
...result
} = datasetDeepSearch
} = datasetDeepSearch && !!text.trim()
? await deepRagSearch({
...searchData,
datasetDeepSearchModel,
......@@ -123,6 +146,13 @@ async function handler(
embeddingTokens: queryExtensionResult.embeddingTokens,
embeddingModel: dataset.vectorModel
}
: undefined,
imageCaptionUsage: imageCaptionResult
? {
model: imageCaptionResult.model,
inputTokens: imageCaptionResult.inputTokens,
outputTokens: imageCaptionResult.outputTokens
}
: undefined
});
......
projects/app/src/service/common/system/index.ts
......@@ -12,7 +12,7 @@ import { POST } from '@fastgpt/service/common/api/plusRequest';
import {
type DeepRagSearchProps,
type SearchDatasetDataResponse
} from '@fastgpt/service/core/dataset/search/controller';
} from '@fastgpt/service/core/dataset/search';
import { type AuthOpenApiLimitProps } from '@fastgpt/service/support/openapi/auth';
import type {
PushUsageItemsProps,
......@@ -219,14 +219,15 @@ export async function initAppTemplateTypes() {
try {
await Promise.all(
defaultTemplateTypes.map((templateType) => {
const { typeOrder, ...rest } = templateType;
return MongoTemplateTypes.updateOne(
{
typeId: templateType.typeId
},
{
$set: rest
$set: {
typeId: templateType.typeId,
typeName: templateType.typeName
}
},
{
upsert: true
......
projects/app/src/service/support/wallet/usage/push.ts
......@@ -280,7 +280,8 @@ export const pushDatasetTestUsage = ({
source = UsageSourceEnum.fastgpt,
embUsage,
rerankUsage,
extensionUsage
extensionUsage,
imageCaptionUsage
}: {
teamId: string;
tmbId: string;
......@@ -300,6 +301,11 @@ export const pushDatasetTestUsage = ({
embeddingTokens: number;
embeddingModel: string;
};
imageCaptionUsage?: {
model: string;
inputTokens: number;
outputTokens: number;
};
}) => {
const list: UsageItemType[] = [];
let points = 0;
......@@ -359,6 +365,21 @@ export const pushDatasetTestUsage = ({
inputTokens: rerankUsage.inputTokens
});
}
if (imageCaptionUsage) {
const { totalPoints, modelName } = formatModelChars2Points({
model: imageCaptionUsage.model,
inputTokens: imageCaptionUsage.inputTokens,
outputTokens: imageCaptionUsage.outputTokens
});
points += totalPoints;
list.push({
moduleName: i18nT('account_usage:image_parse'),
amount: totalPoints,
model: modelName,
inputTokens: imageCaptionUsage.inputTokens,
outputTokens: imageCaptionUsage.outputTokens
});
}
createUsage({
teamId,
......
projects/app/src/web/core/workflow/adapt.ts
......@@ -475,7 +475,20 @@ export const v1Workflow2V2 = (
// 更新特殊的输入(输入全部从开始取)
newNodes.forEach((node) => {
node.inputs.forEach((input) => {
if (workflowStart && input.key === NodeInputKeyEnum.userChatInput) {
if (workflowStart) {
if (
node.flowNodeType === FlowNodeTypeEnum.datasetSearchNode &&
input.key === NodeInputKeyEnum.datasetSearchInput
) {
input.value = [
[workflowStart.nodeId, NodeOutputKeyEnum.userChatInput],
[workflowStart.nodeId, NodeOutputKeyEnum.userFiles]
];
input.valueType = WorkflowIOValueTypeEnum.arrayString;
return;
}
if (input.key !== NodeInputKeyEnum.userChatInput) return;
input.value = [workflowStart.nodeId, NodeOutputKeyEnum.userChatInput];
}
});
......
projects/app/src/web/core/workflow/utils.ts
......@@ -129,9 +129,15 @@ export const storeNode2FlowNode = ({
storeNode.inputs
.filter((item) => !templateInputs.find((input) => input.key === item.key))
.map((item) => {
if (!dynamicInput) return item;
const templateInput = template.inputs.find((input) => input.key === item.key);
if (!dynamicInput) {
return {
...item,
deprecated: templateInput?.deprecated
};
}
return {
...item,
...getInputComponentProps(dynamicInput),
......
projects/app/test/pages/api/core/dataset/searchTest.test.ts 0 → 100644
import { beforeEach, describe, expect, it, vi } from 'vitest';
import { DatasetSearchModeEnum, DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
const mockAuthDataset = vi.hoisted(() => vi.fn());
const mockCheckTeamAIPoints = vi.hoisted(() => vi.fn());
const mockDefaultSearchDatasetData = vi.hoisted(() => vi.fn());
const mockDeepRagSearch = vi.hoisted(() => vi.fn());
const mockPushDatasetTestUsage = vi.hoisted(() => vi.fn());
const mockUpdateApiKeyUsage = vi.hoisted(() => vi.fn());
const mockGetRerankModel = vi.hoisted(() => vi.fn());
const mockAddAuditLog = vi.hoisted(() => vi.fn());
const mockCreateExternalUrl = vi.hoisted(() => vi.fn());
vi.mock('@fastgpt/service/support/permission/dataset/auth', () => ({
authDataset: mockAuthDataset
}));
vi.mock('@fastgpt/service/support/permission/teamLimit', () => ({
checkTeamAIPoints: mockCheckTeamAIPoints
}));
vi.mock('@fastgpt/service/core/dataset/search', () => ({
defaultSearchDatasetData: mockDefaultSearchDatasetData,
deepRagSearch: mockDeepRagSearch
}));
vi.mock('@/service/support/wallet/usage/push', () => ({
pushDatasetTestUsage: mockPushDatasetTestUsage
}));
vi.mock('@fastgpt/service/support/openapi/tools', () => ({
updateApiKeyUsage: mockUpdateApiKeyUsage
}));
vi.mock('@fastgpt/service/core/ai/model', () => ({
getRerankModel: mockGetRerankModel
}));
vi.mock('@fastgpt/service/support/user/audit/util', () => ({
addAuditLog: mockAddAuditLog,
getI18nDatasetType: vi.fn((type: string) => type)
}));
vi.mock('@fastgpt/service/common/s3/sources/dataset', () => ({
getS3DatasetSource: () => ({
createExternalUrl: mockCreateExternalUrl
})
}));
vi.mock('@/service/middleware/entry', () => ({
NextAPI: vi.fn((...args: unknown[]) => args.at(-1))
}));
vi.mock('@fastgpt/service/common/middle/reqFrequencyLimit', () => ({
useIPFrequencyLimit: vi.fn((props: unknown) => props)
}));
import { handler } from '@/pages/api/core/dataset/searchTest';
const datasetId = '507f1f77bcf86cd799439011';
describe('searchTest query image auth', () => {
beforeEach(() => {
vi.clearAllMocks();
mockAuthDataset.mockResolvedValue({
dataset: {
name: 'dataset',
type: DatasetTypeEnum.dataset,
vectorModel: 'mock-vector-model',
vlmModel: 'mock-vlm-model'
},
teamId: 'team-1',
tmbId: 'tmb-1',
userId: 'user-1'
});
mockCheckTeamAIPoints.mockResolvedValue(undefined);
mockGetRerankModel.mockReturnValue({
model: 'mock-rerank-model'
});
mockPushDatasetTestUsage.mockReturnValue({
totalPoints: 0
});
mockDefaultSearchDatasetData.mockResolvedValue({
searchRes: [],
embeddingTokens: 0,
reRankInputTokens: 0,
usingReRank: false,
limit: 5000,
searchMode: DatasetSearchModeEnum.embedding,
similarity: 0
});
mockCreateExternalUrl.mockResolvedValue({
url: 'https://file.fastgpt.io/temp/team-1/search-image.png?token=mock'
});
});
it('should convert current-team temp image keys to external urls before dataset search', async () => {
await handler({
body: {
datasetId,
queryImageUrls: ['temp/team-1/search-image.png']
}
} as any);
expect(mockDefaultSearchDatasetData).toHaveBeenCalledWith(
expect.objectContaining({
teamId: 'team-1',
datasetIds: [datasetId],
textQueries: [],
imageQueries: ['https://file.fastgpt.io/temp/team-1/search-image.png?token=mock']
})
);
expect(mockCreateExternalUrl).toHaveBeenCalledWith({
key: 'temp/team-1/search-image.png',
expiredHours: 1
});
});
it('should reject non-temp or foreign-team query image keys before dataset search', async () => {
await expect(
handler({
body: {
datasetId,
queryImageUrls: [
'temp/team-2/search-image.png',
'dataset/dataset-1/image.png',
'chat/app-1/user-1/chat-1/image.png',
'https://example.com/image.png'
]
}
} as any)
).rejects.toBe('Invalid query image key');
expect(mockDefaultSearchDatasetData).not.toHaveBeenCalled();
expect(mockDeepRagSearch).not.toHaveBeenCalled();
expect(mockCreateExternalUrl).not.toHaveBeenCalled();
});
});
projects/app/test/service/core/dataset/data/data.test.ts
......@@ -15,7 +15,7 @@ import type {
DatasetDataItemType
} from '@fastgpt/global/core/dataset/type';
import { getRootUser } from '@test/datas/users';
import { createMockVectorsResponse, mockGetVectorsByText } from '@test/mocks/core/ai/embedding';
import { createMockVectorsResponse, mockGetVectors } from '@test/mocks/core/ai/embedding';
import { mockVectorDelete, mockVectorInsert, resetVectorMocks } from '@test/mocks/common/vector';
import {
createDatasetData,
......@@ -135,12 +135,12 @@ const toDataItem = (
describe('Dataset data service', () => {
beforeEach(() => {
resetVectorMocks();
mockGetVectorsByText.mockClear();
mockGetVectors.mockClear();
mockDeleteDatasetFileByKey.mockReset();
mockCountPromptTokens.mockClear();
vi.mocked(getEmbeddingModel).mockReturnValue(embeddingModel);
mockGetVectorsByText.mockImplementation(async ({ input }) =>
createMockVectorsResponse(Array.isArray(input) ? input : [input])
mockGetVectors.mockImplementation(async ({ inputs }) =>
createMockVectorsResponse(inputs.map((input) => input.input))
);
mockVectorInsert.mockResolvedValue({
insertIds: ['id_1', 'id_2', 'id_3', 'id_4', 'id_5', 'id_6']
......
projects/app/test/service/core/dataset/data/dataIndex.test.ts
......@@ -11,7 +11,7 @@ import type {
DatasetDataItemType
} from '@fastgpt/global/core/dataset/type';
import { getRootUser } from '@test/datas/users';
import { mockGetVectorsByText, createMockVectorsResponse } from '@test/mocks/core/ai/embedding';
import { mockGetVectors, createMockVectorsResponse } from '@test/mocks/core/ai/embedding';
import { mockVectorDelete, mockVectorInsert, resetVectorMocks } from '@test/mocks/common/vector';
import {
createDatasetDataIndex,
......@@ -102,11 +102,11 @@ const createData = async (
describe('DatasetDataIndexOperation', () => {
beforeEach(() => {
resetVectorMocks();
mockGetVectorsByText.mockClear();
mockGetVectors.mockClear();
mockCountPromptTokens.mockClear();
vi.mocked(getEmbeddingModel).mockReturnValue(embeddingModel);
mockGetVectorsByText.mockImplementation(async ({ input }) =>
createMockVectorsResponse(Array.isArray(input) ? input : [input])
mockGetVectors.mockImplementation(async ({ inputs }) =>
createMockVectorsResponse(inputs.map((input) => input.input))
);
mockVectorInsert.mockResolvedValue({
insertIds: ['id_1', 'id_2', 'id_3', 'id_4', 'id_5']
......
projects/app/test/web/core/app/workflow/store2flow.deprecated.test.ts
......@@ -75,7 +75,7 @@ describe('storeNode2FlowNode with deprecated inputs/outputs', () => {
const deprecatedInput = result.data.inputs.find((input) => input.key === 'deprecatedInput');
expect(deprecatedInput).toBeDefined();
expect(deprecatedInput?.deprecated).toBe(undefined);
expect(deprecatedInput?.deprecated).toBe(true);
const deprecatedOutput = result.data.outputs.find(
(output) => output.key === 'deprecatedOutput'
......
test/mocks/core/ai/embedding.ts
......@@ -39,7 +39,7 @@ export const generateMockEmbeddings = (texts: string[], dimension: number = 1536
};
/**
* Create a mock response for getVectorsByText
* Create a mock response for getVectors
*/
export const createMockVectorsResponse = (
texts: string | string[],
......@@ -88,19 +88,18 @@ export const generateOrthogonalVector = (baseVector: number[]): number[] => {
};
/**
* Mock implementation for getVectorsByText
* Automatically generates embeddings based on input text
* Mock implementation for getVectors
* Automatically generates embeddings based on input content
*/
export const mockGetVectorsByText = vi.fn(
export const mockGetVectors = vi.fn(
async ({
input,
type
inputs
}: {
model: any;
input: string[] | string;
inputs: { type: 'text' | 'image'; input: string }[];
type?: string;
}): Promise<{ tokens: number; vectors: number[][] }> => {
const texts = Array.isArray(input) ? input : [input];
const texts = inputs.map((input) => input.input);
return createMockVectorsResponse(texts);
}
);
......@@ -112,7 +111,7 @@ vi.mock('@fastgpt/service/core/ai/embedding', async (importOriginal) => {
const actual = (await importOriginal()) as any;
return {
...actual,
getVectorsByText: mockGetVectorsByText
getVectors: mockGetVectors
};
});
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment