Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit 3f891a2e by YeYuheng Committed by GitHub
Options

feat: support image dataset search (#6942) 

* feat: support image dataset search

* image recall code

* fix: normalize dataset image inputs

* fix: handle dataset image preview expiry

* update vitest

* fix: track image caption usage metadata

* fix: filter blank embedding queries at callers

* fix: normalize blank dataset search queries

* doc

---------

Co-authored-by: archer <545436317@qq.com>
parent 92e69588
Showing with 3308 additions and 333 deletions
document/content/self-host/upgrading/4-15/41502.mdx
......@@ -6,6 +6,7 @@ description: 'FastGPT V4.15.0-beta2 更新说明'
## 🚀 新增内容
1. 重写 agentV2 loop 逻辑。
2. 知识库搜索支持原生多模态 embedding 模型以及图搜图。
## ⚙️ 优化
......
document/data/doc-last-modified.json
......@@ -59,8 +59,8 @@
"content/guide/build/workflow/nodes/ai_chat.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/content_extract.en.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/content_extract.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/coreferenceResolution.en.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/coreferenceResolution.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/coreferenceResolution.en.mdx": "2026-05-18T17:26:16+08:00",
"content/guide/build/workflow/nodes/coreferenceResolution.mdx": "2026-05-18T17:26:16+08:00",
"content/guide/build/workflow/nodes/custom_feedback.en.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/custom_feedback.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/dataset_search.en.mdx": "2026-05-07T15:06:40+08:00",
......@@ -69,8 +69,8 @@
"content/guide/build/workflow/nodes/document_parsing.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/form_input.en.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/form_input.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/http.en.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/http.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/http.en.mdx": "2026-05-18T17:26:16+08:00",
"content/guide/build/workflow/nodes/http.mdx": "2026-05-18T17:26:16+08:00",
"content/guide/build/workflow/nodes/knowledge_base_search_merge.en.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/knowledge_base_search_merge.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/laf.en.mdx": "2026-05-07T15:06:40+08:00",
......@@ -85,12 +85,12 @@
"content/guide/build/workflow/nodes/reply.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/sandbox-v2.en.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/sandbox-v2.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/text_editor.en.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/text_editor.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/text_editor.en.mdx": "2026-05-18T17:26:16+08:00",
"content/guide/build/workflow/nodes/text_editor.mdx": "2026-05-18T17:26:16+08:00",
"content/guide/build/workflow/nodes/tfswitch.en.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/tfswitch.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/tool.en.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/tool.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/tool.en.mdx": "2026-05-18T17:26:16+08:00",
"content/guide/build/workflow/nodes/tool.mdx": "2026-05-18T17:26:16+08:00",
"content/guide/build/workflow/nodes/user-selection.en.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/user-selection.mdx": "2026-05-07T15:06:40+08:00",
"content/guide/build/workflow/nodes/variable_update.en.mdx": "2026-05-07T15:06:40+08:00",
......@@ -274,7 +274,7 @@
"content/self-host/upgrading/4-14/4149.en.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/4-14/4149.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/4-15/4150.mdx": "2026-05-09T16:13:01+08:00",
"content/self-host/upgrading/4-15/41502.mdx": "2026-05-17T18:56:43+08:00",
"content/self-host/upgrading/4-15/41502.mdx": "2026-05-18T17:26:16+08:00",
"content/self-host/upgrading/outdated/40.en.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/outdated/40.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/outdated/41.en.mdx": "2026-04-26T21:08:47+08:00",
......@@ -415,6 +415,6 @@
"content/self-host/upgrading/outdated/499.mdx": "2026-05-07T15:06:40+08:00",
"content/self-host/upgrading/upgrade-intruction.en.mdx": "2026-04-26T21:08:47+08:00",
"content/self-host/upgrading/upgrade-intruction.mdx": "2026-04-26T21:08:47+08:00",
"content/toc.en.mdx": "2026-05-14T17:58:58+08:00",
"content/toc.mdx": "2026-05-14T17:58:58+08:00"
"content/toc.en.mdx": "2026-05-18T17:26:16+08:00",
"content/toc.mdx": "2026-05-18T17:26:16+08:00"
}
\ No newline at end of file
eslint.config.mjs
......@@ -20,6 +20,7 @@ export default defineConfig([
files: ['**/*.{ts,tsx}'],
rules: {
'@typescript-eslint/no-explicit-any': 'off',
'@typescript-eslint/no-non-null-asserted-optional-chain': 'off',
'@typescript-eslint/consistent-type-imports': [
'error',
{
......
packages/global/core/workflow/constants.ts
......@@ -190,6 +190,7 @@ export enum NodeInputKeyEnum {
datasetSearchUsingExtensionQuery = 'datasetSearchUsingExtensionQuery',
datasetSearchExtensionModel = 'datasetSearchExtensionModel',
datasetSearchExtensionBg = 'datasetSearchExtensionBg',
datasetSearchInput = 'datasetSearchInput',
collectionFilterMatch = 'collectionFilterMatch',
authTmbId = 'authTmbId',
datasetDeepSearch = 'datasetDeepSearch',
......
packages/global/core/workflow/runtime/type.ts
......@@ -220,17 +220,8 @@ export const DispatchNodeResponseSchema = z
limit: z.number().optional().meta({ description: '限制' }),
searchMode: z.enum(DatasetSearchModeEnum).optional().meta({ description: '搜索模式' }),
embeddingWeight: z.number().optional().meta({ description: '嵌入权重' }),
filteredFileCount: z.number().optional().meta({ description: '过滤的非图片文件数量' }),
queryImages: z
.array(
z.object({
key: z.string().optional(),
url: z.string().optional(),
name: z.string().optional()
})
)
.optional()
.meta({ description: '参与知识库检索的图片' }),
datasetQueries: z.array(z.string()).optional().meta({ description: '检索词' }),
rerankModel: z.string().optional().meta({ description: '重排模型' }),
rerankWeight: z.number().optional().meta({ description: '重排权重' }),
reRankInputTokens: z.number().optional().meta({ description: '重排输入 token' }),
......
packages/global/core/workflow/template/system/datasetSearch.ts
......@@ -126,6 +126,14 @@ export const DatasetSearchModule: FlowNodeTemplateType = {
},
{
...Input_Template_UserChatInput,
toolDescription: i18nT('workflow:content_to_search'),
deprecated: true
},
{
...Input_Template_UserChatInput,
label: i18nT('workflow:search_query'),
key: NodeInputKeyEnum.datasetSearchInput,
valueType: WorkflowIOValueTypeEnum.arrayString,
toolDescription: i18nT('workflow:content_to_search')
},
{
......
packages/global/test/core/workflow/template/datasetSearch.test.ts 0 → 100644
import { describe, expect, it } from 'vitest';
import { NodeInputKeyEnum, WorkflowIOValueTypeEnum } from '@fastgpt/global/core/workflow/constants';
import { DatasetSearchModule } from '@fastgpt/global/core/workflow/template/system/datasetSearch';
describe('DatasetSearchModule', () => {
it('should deprecate legacy user question input and add array search input', () => {
const legacyUserQuestionInput = DatasetSearchModule.inputs.find(
(input) => input.key === NodeInputKeyEnum.userChatInput
);
const searchInput = DatasetSearchModule.inputs.find(
(input) => input.key === NodeInputKeyEnum.datasetSearchInput
);
expect(legacyUserQuestionInput?.valueType).toBe(WorkflowIOValueTypeEnum.string);
expect(legacyUserQuestionInput?.deprecated).toBe(true);
expect(searchInput?.valueType).toBe(WorkflowIOValueTypeEnum.arrayString);
});
});
packages/service/common/api/type.d.ts
import type {
DeepRagSearchProps,
SearchDatasetDataResponse
} from '../../core/dataset/search/controller';
import type { DeepRagSearchProps, SearchDatasetDataResponse } from '../../core/dataset/search';
import type { AuthOpenApiLimitProps } from '../../support/openapi/auth';
import type {
CreateUsageProps,
......
packages/service/common/vectorDB/controller.ts
......@@ -3,7 +3,7 @@ import { PgVectorCtrl } from './pg';
import { ObVectorCtrl } from './oceanbase';
import { SeekVectorCtrl } from './seekdb';
import { OpenGaussVectorCtrl } from './opengauss';
import { getVectorsByText } from '../../core/ai/embedding';
import { getVectors } from '../../core/ai/embedding';
import type { VectorControllerType, InsertVectorControllerPropsType } from './type';
import { type EmbeddingModelItemType } from '@fastgpt/global/core/ai/model.schema';
import {
......@@ -111,9 +111,19 @@ export const insertDatasetDataVector = async ({
inputs: string[];
model: EmbeddingModelItemType;
}) => {
const { vectors, tokens } = await getVectorsByText({
if (inputs.length === 0) {
return {
tokens: 0,
insertIds: []
};
}
const { vectors, tokens } = await getVectors({
model,
input: inputs,
inputs: inputs.map((text) => ({
type: 'text',
input: text
})),
type: 'db'
});
const { insertIds } = await retryFn(() =>
......
packages/service/core/ai/embedding/index.ts
......@@ -4,34 +4,67 @@ import { countPromptTokens } from '../../../common/string/tiktoken/index';
import { EmbeddingTypeEnm } from '@fastgpt/global/core/ai/constants';
import { retryFn } from '@fastgpt/global/common/system/utils';
import { getLogger, LogCategories } from '../../../common/logger';
import z from 'zod';
const logger = getLogger(LogCategories.MODULE.AI.EMBEDDING);
type GetVectorProps = {
type GetVectorsBaseProps = {
model: EmbeddingModelItemType;
input: string[] | string;
type?: `${EmbeddingTypeEnm}`;
headers?: Record<string, string>;
};
// text to vector
export async function getVectorsByText({ model, input, type, headers }: GetVectorProps) {
if (!input) {
const InputItemSchema = z.object({
type: z.enum(['text', 'image']),
input: z.string()
});
type GetVectorInputItem = z.infer<typeof InputItemSchema>;
export type GetVectorsProps = GetVectorsBaseProps & {
inputs: GetVectorInputItem[];
};
const getRequestInput = (input: GetVectorInputItem) => {
if (input.type === 'image') {
return {
type: 'image_url',
image_url: {
url: input.input
}
};
}
return input.input;
};
const countInputTokens = async (input: GetVectorInputItem) => {
if (input.type === 'image') return 1;
return countPromptTokens(input.input);
};
export async function getVectors({ model, inputs: rawInputs, type, headers }: GetVectorsProps) {
const inputs = z
.array(InputItemSchema)
.parse(rawInputs)
.map((item) => ({
...item,
input: item.input.trim()
}));
if (inputs.length === 0 || inputs.some((item) => !item.input)) {
return Promise.reject({
code: 500,
message: 'input is empty'
});
}
const { ai } = getAIApi();
const formatInput = Array.isArray(input) ? input : [input];
const { ai } = getAIApi();
let chunkSize = Number(model.batchSize || 1);
chunkSize = isNaN(chunkSize) ? 1 : chunkSize;
const chunks = [];
for (let i = 0; i < formatInput.length; i += chunkSize) {
chunks.push(formatInput.slice(i, i + chunkSize));
for (let i = 0; i < inputs.length; i += chunkSize) {
chunks.push(inputs.slice(i, i + chunkSize));
}
try {
......@@ -40,18 +73,20 @@ export async function getVectorsByText({ model, input, type, headers }: GetVecto
const allVectors: number[][] = [];
for (const chunk of chunks) {
// input text to vector
const requestInput = chunk.map(getRequestInput);
const inputTypes = Array.from(new Set(chunk.map((item) => item.type)));
const result = await retryFn(() =>
ai.embeddings
.create(
{
model: model.model,
input: chunk,
input: requestInput,
encoding_format: 'float',
...model.defaultConfig,
...(type === EmbeddingTypeEnm.db && model.dbConfig),
...(type === EmbeddingTypeEnm.query && model.queryConfig)
},
} as any,
model.requestUrl
? {
path: model.requestUrl,
......@@ -66,17 +101,19 @@ export async function getVectorsByText({ model, input, type, headers }: GetVecto
if (!res.data) {
logger.error('Embedding API returned empty data', {
model: model.model,
inputLength: chunk.length,
inputTypes,
inputCount: chunk.length,
response: res
});
return Promise.reject('Embedding API is not responding');
}
if (!res?.data?.[0]?.embedding) {
// @ts-ignore
// @ts-expect-error provider error payload is not part of the embedding response type
const msg = res.data?.err?.message || '';
logger.error('Embedding API returned invalid embedding', {
model: model.model,
inputLength: chunk.length,
inputTypes,
inputCount: chunk.length,
response: res,
apiMessage: msg
});
......@@ -87,7 +124,7 @@ export async function getVectorsByText({ model, input, type, headers }: GetVecto
(async () => {
if (res.usage) return res.usage.total_tokens;
const tokens = await Promise.all(chunk.map((item) => countPromptTokens(item)));
const tokens = await Promise.all(chunk.map(countInputTokens));
return tokens.reduce((sum, item) => sum + item, 0);
})(),
Promise.all(
......@@ -115,7 +152,8 @@ export async function getVectorsByText({ model, input, type, headers }: GetVecto
} catch (error) {
logger.error('Embedding request failed', {
model: model.model,
inputLengths: formatInput.map((item) => item.length),
inputTypes: Array.from(new Set(inputs.map((item) => item.type))),
inputCount: inputs.length,
error
});
......
packages/service/core/ai/functions/queryExtension.ts
......@@ -250,7 +250,21 @@ export const queryExtension = async ({
const { lazyGreedyQuerySelection, embeddingModel: useEmbeddingModel } = useTextCosine({
embeddingModel
});
queries = queries.map((item) => String(item));
queries = queries.map((item) => String(item).trim()).filter(Boolean);
if (queries.length === 0) {
return {
rawQuery: query,
extensionQueries: [],
llmModel: modelData.model,
embeddingModel,
requestId,
seconds,
inputTokens,
outputTokens,
usedUserOpenAIKey,
embeddingTokens: 0
};
}
const { selectedData: selectedQueries, embeddingTokens } = await lazyGreedyQuerySelection({
originalText: query,
......
packages/service/core/ai/hooks/useTextCosine.ts
......@@ -3,7 +3,7 @@
Reference: https://github.com/jina-ai/submodular-optimization
*/
import { getVectorsByText } from '../embedding';
import { getVectors } from '../embedding';
import { getEmbeddingModel } from '../model';
class PriorityQueue<T> {
......@@ -83,16 +83,28 @@ export const useTextCosine = ({ embeddingModel }: { embeddingModel: string }) =>
k: number;
alpha?: number;
}) => {
const { tokens: embeddingTokens, vectors: embeddingVectors } = await getVectorsByText({
const query = originalText.trim();
const normalizedCandidates = candidates.map((item) => item.trim()).filter(Boolean);
if (!query || normalizedCandidates.length === 0 || k <= 0) {
return {
selectedData: [],
embeddingTokens: 0
};
}
const { tokens: embeddingTokens, vectors: embeddingVectors } = await getVectors({
model: vectorModel,
input: [originalText, ...candidates],
inputs: [query, ...normalizedCandidates].map((text) => ({
type: 'text',
input: text
})),
type: 'query'
});
const originalEmbedding = embeddingVectors[0];
const candidateEmbeddings = embeddingVectors.slice(1);
const n = candidates.length;
const n = normalizedCandidates.length;
const selected: string[] = [];
const selectedEmbeddings: number[][] = [];
......@@ -136,7 +148,7 @@ export const useTextCosine = ({ embeddingModel }: { embeddingModel: string }) =>
}
if (bestCandidate) {
selected.push(candidates[bestCandidate.index]);
selected.push(normalizedCandidates[bestCandidate.index]);
selectedEmbeddings.push(candidateEmbeddings[bestCandidate.index]);
}
}
......
packages/service/core/ai/model.ts
import { cloneDeep } from 'lodash';
import { type SystemModelItemType } from './type';
import type { LLMModelItemType } from '@fastgpt/global/core/ai/model.schema';
import type {
EmbeddingModelItemType,
LLMModelItemType
} from '@fastgpt/global/core/ai/model.schema';
export const getDefaultLLMModel = () => global?.systemDefaultModel.llm!;
export const getDefaultLLMModel = () => global.systemDefaultModel.llm!;
export const getLLMModel = (model?: string | LLMModelItemType) => {
if (!model) return getDefaultLLMModel();
......@@ -30,9 +33,16 @@ export const getDefaultHelperBotModel = (): LLMModelItemType =>
global?.systemDefaultModel.helperBotLLM || getDefaultLLMModel();
export const getDefaultEmbeddingModel = () => global?.systemDefaultModel.embedding!;
export const getEmbeddingModel = (model?: string) => {
export const getEmbeddingModel = (model?: string | EmbeddingModelItemType) => {
if (!model) return getDefaultEmbeddingModel();
return global.embeddingModelMap.get(model) || getDefaultEmbeddingModel();
if (typeof model === 'string') {
return global.embeddingModelMap.get(model) || getDefaultEmbeddingModel();
}
return model;
};
export const isImageEmbeddingModel = (model?: string | EmbeddingModelItemType) => {
return !!getEmbeddingModel(model)?.vision;
};
export const getDefaultTTSModel = () => global?.systemDefaultModel.tts!;
......
packages/service/core/dataset/search/defaultRecall/collectionFilter.ts 0 → 100644
import json5 from 'json5';
import { MongoDatasetCollection } from '../../collection/schema';
import { MongoDatasetCollectionTags } from '../../tag/schema';
import { readFromSecondary } from '../../../../common/mongo/utils';
import { computeFilterIntersection } from '../utils';
export const getForbidCollectionIdList = async ({
teamId,
datasetIds
}: {
teamId: string;
datasetIds: string[];
}) => {
const collections = await MongoDatasetCollection.find(
{
teamId,
datasetId: { $in: datasetIds },
forbid: true
},
'_id'
);
return collections.map((item) => String(item._id));
};
/**
* 按知识库集合元数据过滤 collectionId。
*
* 标签过滤保持原有语义：`$and` 优先生效，且 `$and` 中字符串标签和 null 不能共存。
* 输入 collectionIds 可以是文件夹，会递归展开为实际文件集合。
*/
export const filterCollectionByMetadata = async ({
teamId,
datasetIds,
collectionFilterMatch
}: {
teamId: string;
datasetIds: string[];
collectionFilterMatch?: string;
}): Promise<string[] | undefined> => {
const getAllCollectionIds = async ({
parentCollectionIds
}: {
parentCollectionIds?: string[];
}): Promise<string[] | undefined> => {
if (!parentCollectionIds) return;
if (parentCollectionIds.length === 0) {
return [];
}
const collections = await MongoDatasetCollection.find(
{
teamId,
datasetId: { $in: datasetIds },
_id: { $in: parentCollectionIds }
},
'_id type',
{
...readFromSecondary
}
).lean();
const resultIds = new Set<string>();
collections.forEach((item) => {
if (item.type !== 'folder') {
resultIds.add(String(item._id));
}
});
const folderIds = collections
.filter((item) => item.type === 'folder')
.map((item) => String(item._id));
// Get all child collection ids
if (folderIds.length) {
const childCollections = await MongoDatasetCollection.find(
{
teamId,
datasetId: { $in: datasetIds },
parentId: { $in: folderIds }
},
'_id type',
{
...readFromSecondary
}
).lean();
const childIds = await getAllCollectionIds({
parentCollectionIds: childCollections.map((item) => String(item._id))
});
childIds?.forEach((id) => resultIds.add(id));
}
return Array.from(resultIds);
};
if (!collectionFilterMatch || !global.feConfigs.isPlus) return;
let tagCollectionIdList: string[] | undefined = undefined;
let createTimeCollectionIdList: string[] | undefined = undefined;
let inputCollectionIdList: string[] | undefined = undefined;
try {
const jsonMatch = json5.parse(collectionFilterMatch);
const andTags = jsonMatch?.tags?.$and as (string | null)[] | undefined;
const orTags = jsonMatch?.tags?.$or as (string | null)[] | undefined;
if (andTags && andTags.length > 0) {
const uniqueAndTags = Array.from(new Set(andTags));
if (uniqueAndTags.includes(null) && uniqueAndTags.some((tag) => typeof tag === 'string')) {
return [];
}
if (uniqueAndTags.every((tag) => typeof tag === 'string')) {
const matchedTags = await MongoDatasetCollectionTags.find(
{
teamId,
datasetId: { $in: datasetIds },
tag: { $in: uniqueAndTags as string[] }
},
'_id datasetId tag',
{ ...readFromSecondary }
).lean();
// Group tags by dataset
const datasetTagMap = new Map<string, { tagIds: string[]; tagNames: Set<string> }>();
matchedTags.forEach((tag) => {
const datasetId = String(tag.datasetId);
if (!datasetTagMap.has(datasetId)) {
datasetTagMap.set(datasetId, {
tagIds: [],
tagNames: new Set()
});
}
const datasetData = datasetTagMap.get(datasetId)!;
datasetData.tagIds.push(String(tag._id));
datasetData.tagNames.add(tag.tag);
});
const validDatasetIds = Array.from(datasetTagMap.entries())
.filter(([, data]) => uniqueAndTags.every((tag) => data.tagNames.has(tag as string)))
.map(([datasetId]) => datasetId);
if (validDatasetIds.length === 0) return [];
const collectionsPromises = validDatasetIds.map((datasetId) => {
const { tagIds } = datasetTagMap.get(datasetId)!;
return MongoDatasetCollection.find(
{
teamId,
datasetId,
tags: { $all: tagIds }
},
'_id',
{ ...readFromSecondary }
).lean();
});
const collectionsResults = await Promise.all(collectionsPromises);
tagCollectionIdList = collectionsResults.flat().map((item) => String(item._id));
} else if (uniqueAndTags.every((tag) => tag === null)) {
const collections = await MongoDatasetCollection.find(
{
teamId,
datasetId: { $in: datasetIds },
$or: [{ tags: { $size: 0 } }, { tags: { $exists: false } }]
},
'_id',
{ ...readFromSecondary }
).lean();
tagCollectionIdList = collections.map((item) => String(item._id));
}
} else if (orTags && orTags.length > 0) {
// Get tagId by tag string
const orTagArray = await MongoDatasetCollectionTags.find(
{
teamId,
datasetId: { $in: datasetIds },
tag: { $in: orTags.filter((tag) => tag !== null) }
},
'_id',
{ ...readFromSecondary }
).lean();
const orTagIds = orTagArray.map((item) => String(item._id));
// Get collections by tagId
const collections = await MongoDatasetCollection.find(
{
teamId,
datasetId: { $in: datasetIds },
$or: [
{ tags: { $in: orTagIds } },
...(orTags.includes(null) ? [{ tags: { $size: 0 } }] : [])
]
},
'_id',
{ ...readFromSecondary }
).lean();
tagCollectionIdList = collections.map((item) => String(item._id));
}
// time
const getCreateTime = jsonMatch?.createTime?.$gte as string | undefined;
const lteCreateTime = jsonMatch?.createTime?.$lte as string | undefined;
if (getCreateTime || lteCreateTime) {
const collections = await MongoDatasetCollection.find(
{
teamId,
datasetId: { $in: datasetIds },
createTime: {
...(getCreateTime && { $gte: new Date(getCreateTime) }),
...(lteCreateTime && {
$lte: new Date(lteCreateTime)
})
}
},
'_id'
);
createTimeCollectionIdList = collections.map((item) => String(item._id));
}
// collectionIds
const inputCollectionIds = jsonMatch?.collectionIds as string[] | undefined;
if (Array.isArray(inputCollectionIds) && inputCollectionIds.length > 0) {
inputCollectionIdList = await getAllCollectionIds({
parentCollectionIds: inputCollectionIds
});
if (inputCollectionIdList && inputCollectionIdList.length === 0) {
return [];
}
}
// Concat tag, time and collectionIds
const collectionIds = computeFilterIntersection([
tagCollectionIdList,
createTimeCollectionIdList,
inputCollectionIdList
]);
return await getAllCollectionIds({
parentCollectionIds: collectionIds
});
} catch {}
};
packages/service/core/dataset/search/defaultRecall/constant.ts 0 → 100644
/**
* 默认召回只读取生成搜索结果所需字段。
* 使用对象投影可以让字段增删更明确，也避免字符串投影在重构时难以 diff。
*/
export const datasetDataSelectField = {
_id: 1,
datasetId: 1,
collectionId: 1,
updateTime: 1,
q: 1,
a: 1,
imageId: 1,
imageDescMap: 1,
chunkIndex: 1,
indexes: 1
};
/**
* collection 只需要来源展示字段。
* 文件内容、权限等重字段不在召回阶段读取，减少搜索路径的 Mongo 负担。
*/
export const datasetCollectionSelectField = {
_id: 1,
name: 1,
fileId: 1,
rawLink: 1,
apiFileId: 1,
externalFileId: 1,
externalFileUrl: 1
};
packages/service/core/dataset/search/defaultRecall/embeddingRecall.ts 0 → 100644
import { SearchScoreTypeEnum } from '@fastgpt/global/core/dataset/constants';
import type {
DatasetCollectionSchemaType,
DatasetDataSchemaType,
SearchDataResponseItemType
} from '@fastgpt/global/core/dataset/type';
import { recallFromVectorStore } from '../../../../common/vectorDB/controller';
import { getVectors } from '../../../ai/embedding';
import { getEmbeddingModel, isImageEmbeddingModel } from '../../../ai/model';
import { MongoDatasetCollection } from '../../collection/schema';
import { MongoDatasetData } from '../../data/schema';
import { getLogger, LogCategories } from '../../../../common/logger';
import { readFromSecondary } from '../../../../common/mongo/utils';
import { normalizeImageToBase64 } from '../utils';
import { datasetCollectionSelectField, datasetDataSelectField } from './constant';
import { buildSearchResultItem, concatRecallLists } from './result';
const logger = getLogger(LogCategories.MODULE.DATASET.DATA);
type EmbeddingRecallSource = 'text' | 'imageCaption' | 'image';
type VectorRecallTask = {
source: EmbeddingRecallSource;
vector: number[];
};
const emptyEmbeddingRecallResult = () => ({
textEmbeddingRecallResults: [] as SearchDataResponseItemType[],
imageCaptionEmbeddingRecallResults: [] as SearchDataResponseItemType[],
imageVectorRecallResults: [] as SearchDataResponseItemType[]
});
/**
* 构造向量召回任务。
* 文本 query 和图片描述 query 都按 text embedding 处理；原始图片 query 只有在当前
* embedding 模型支持图片时才会转 base64 并参与 image embedding。单张图片解析失败
* 会被跳过，避免影响其他文本或图片任务。
*/
const buildVectorRecallTasks = async ({
model,
textQueries,
imageCaptionQueries,
imageQueries
}: {
model: string;
textQueries: string[];
imageCaptionQueries: string[];
imageQueries: string[];
}): Promise<{
tasks: VectorRecallTask[];
tokens: number;
}> => {
const embeddingModel = getEmbeddingModel(model);
const textTasks = [
...textQueries.map((query) => ({ source: 'text' as const, query })),
...imageCaptionQueries.map((query) => ({ source: 'imageCaption' as const, query }))
]
.map((item) => ({
...item,
query: item.query.trim()
}))
.filter((item) => item.query);
const vectorInputs: {
source: EmbeddingRecallSource;
input: Parameters<typeof getVectors>[0]['inputs'][number];
}[] = textTasks.map((item) => ({
source: item.source,
input: {
type: 'text',
input: item.query
}
}));
const validImageQueries = imageQueries.map((url) => url.trim()).filter(Boolean);
if (validImageQueries.length > 0 && isImageEmbeddingModel(embeddingModel)) {
const imageInputs = (
await Promise.all(
validImageQueries.map(async (url, index) => {
try {
return await normalizeImageToBase64(url);
} catch (error) {
// Image search is additive. A stale or unreadable image should not break text recall
// or other valid images in the same request.
logger.warn('Image embedding normalization failed during dataset search', {
imageIndex: index,
error
});
}
})
)
)
.filter((imageUrl): imageUrl is string => typeof imageUrl === 'string' && !!imageUrl.trim())
.map((imageUrl) => ({
source: 'image' as const,
input: {
type: 'image' as const,
input: imageUrl.trim()
}
}));
vectorInputs.push(...imageInputs);
}
if (vectorInputs.length === 0) {
return {
tasks: [],
tokens: 0
};
}
const { tokens, vectors } = await getVectors({
model: embeddingModel,
inputs: vectorInputs.map((item) => item.input),
type: 'query'
});
const tasks = vectors.map((vector, index) => ({
source: vectorInputs[index].source,
vector
}));
return {
tasks,
tokens
};
};
/**
* 执行 embedding 召回并按 query 来源分组返回。
* 向量库返回的是 index dataId，这里需要再回查 data/collection，补齐 q/a、
* 图片字段和来源信息。每个 query 内先按数据块去重，多个 query 之间再交给 RRF 合并。
*/
export const embeddingRecall = async ({
teamId,
datasetIds,
model,
imageQueries,
textQueries,
imageCaptionQueries,
limit,
forbidCollectionIdList,
filterCollectionIdList
}: {
teamId: string;
datasetIds: string[];
model: string;
imageQueries: string[];
textQueries: string[];
imageCaptionQueries: string[];
limit: number;
forbidCollectionIdList: string[];
filterCollectionIdList?: string[];
}): Promise<
ReturnType<typeof emptyEmbeddingRecallResult> & {
tokens: number;
}
> => {
if (limit === 0) {
return {
...emptyEmbeddingRecallResult(),
tokens: 0
};
}
const { tasks, tokens } = await buildVectorRecallTasks({
model,
textQueries,
imageCaptionQueries,
imageQueries
});
if (tasks.length === 0) {
return {
...emptyEmbeddingRecallResult(),
tokens
};
}
const recallResults = await Promise.all(
tasks.map(async ({ vector }) => {
return await recallFromVectorStore({
teamId,
datasetIds,
vector,
limit,
forbidCollectionIdList,
filterCollectionIdList
});
})
);
const collectionIdList = Array.from(
new Set(recallResults.map((item) => item.results.map((item) => item.collectionId)).flat())
);
const indexDataIds = Array.from(
new Set(recallResults.map((item) => item.results.map((item) => item.id?.trim())).flat())
);
const [dataMaps, collectionMaps] = await Promise.all([
MongoDatasetData.find(
{
teamId,
datasetId: { $in: datasetIds },
collectionId: { $in: collectionIdList },
'indexes.dataId': { $in: indexDataIds }
},
datasetDataSelectField,
{ ...readFromSecondary }
)
.lean()
.then((res) => {
const map = new Map<string, DatasetDataSchemaType>();
res.forEach((item) => {
item.indexes.forEach((index) => {
map.set(String(index.dataId), item);
});
});
return map;
}),
MongoDatasetCollection.find(
{
_id: { $in: collectionIdList }
},
datasetCollectionSelectField,
{ ...readFromSecondary }
)
.lean()
.then((res) => {
const map = new Map<string, DatasetCollectionSchemaType>();
res.forEach((item) => {
map.set(String(item._id), item);
});
return map;
})
]);
const groupedRecallLists: Record<EmbeddingRecallSource, SearchDataResponseItemType[][]> = {
text: [],
imageCaption: [],
image: []
};
recallResults.forEach((recallResult, taskIndex) => {
const task = tasks[taskIndex];
const set = new Set<string>();
const list = recallResult.results
.map((item, index) => {
const collection = collectionMaps.get(String(item.collectionId));
if (!collection) {
logger.warn('Dataset collection not found during recall', {
collectionId: item.collectionId,
dataId: item.id
});
return;
}
const data = dataMaps.get(String(item.id?.trim()));
if (!data) {
logger.warn('Dataset data not found during recall', {
dataId: item.id,
collectionId: item.collectionId
});
return;
}
return buildSearchResultItem({
data,
collection,
score: [{ type: SearchScoreTypeEnum.embedding, value: item?.score || 0, index }]
});
})
.filter((item) => {
if (!item) return false;
if (set.has(item.id)) return false;
set.add(item.id);
return true;
})
.map((item, index) => {
return {
...item!,
score: item!.score.map((item) => ({ ...item, index }))
};
}) as SearchDataResponseItemType[];
groupedRecallLists[task.source].push(list);
});
return {
textEmbeddingRecallResults: concatRecallLists(groupedRecallLists.text, limit),
imageCaptionEmbeddingRecallResults: concatRecallLists(groupedRecallLists.imageCaption, limit),
imageVectorRecallResults: concatRecallLists(groupedRecallLists.image, limit),
tokens
};
};
packages/service/core/dataset/search/defaultRecall/fullTextRecall.ts 0 → 100644
import { SearchScoreTypeEnum } from '@fastgpt/global/core/dataset/constants';
import type {
DatasetCollectionSchemaType,
DatasetDataSchemaType,
DatasetDataTextSchemaType,
SearchDataResponseItemType
} from '@fastgpt/global/core/dataset/type';
import { jiebaSplit } from '../../../../common/string/jieba/index';
import { Types } from '../../../../common/mongo';
import { readFromSecondary } from '../../../../common/mongo/utils';
import { getLogger, LogCategories } from '../../../../common/logger';
import { MongoDatasetCollection } from '../../collection/schema';
import { MongoDatasetDataText } from '../../data/dataTextSchema';
import { MongoDatasetData } from '../../data/schema';
import { datasetCollectionSelectField, datasetDataSelectField } from './constant';
import { buildSearchResultItem, concatRecallLists } from './result';
const logger = getLogger(LogCategories.MODULE.DATASET.DATA);
type FullTextRecallSource = 'text' | 'imageCaption';
/**
* 执行 Mongo full-text 召回并按 query 来源分组返回。
* 目前 full-text 只处理文本类 query：用户文本和图片 caption。原始图片不会进入这里，
* 因为 Mongo 文本索引无法直接消费图片向量或图片 URL。
*/
export const fullTextRecall = async ({
teamId,
datasetIds,
queryGroups,
limit,
filterCollectionIdList,
forbidCollectionIdList
}: {
teamId: string;
datasetIds: string[];
queryGroups: {
source: FullTextRecallSource;
queries: string[];
}[];
limit: number;
filterCollectionIdList?: string[];
forbidCollectionIdList: string[];
}): Promise<{
textFullTextRecallResults: SearchDataResponseItemType[];
imageCaptionFullTextRecallResults: SearchDataResponseItemType[];
}> => {
const queryTasks = queryGroups.flatMap((group) =>
group.queries
.map((query) => query.trim())
.filter(Boolean)
.map((query) => ({ source: group.source, query }))
);
if (limit === 0 || queryTasks.length === 0) {
return {
textFullTextRecallResults: [],
imageCaptionFullTextRecallResults: []
};
}
const recallResults = await Promise.all(
queryTasks.map(async ({ query }) => {
return (await MongoDatasetDataText.aggregate(
[
{
$match: {
teamId: new Types.ObjectId(teamId),
$text: { $search: await jiebaSplit({ text: query }) },
datasetId: { $in: datasetIds.map((id) => new Types.ObjectId(id)) },
...(filterCollectionIdList
? {
collectionId: {
$in: filterCollectionIdList
.filter((id) => !forbidCollectionIdList.includes(id))
.map((id) => new Types.ObjectId(id))
}
}
: forbidCollectionIdList?.length
? {
collectionId: {
$nin: forbidCollectionIdList.map((id) => new Types.ObjectId(id))
}
}
: {})
}
},
{
$sort: {
score: { $meta: 'textScore' }
}
},
{
$limit: limit
},
{
$project: {
_id: 1,
collectionId: 1,
dataId: 1,
score: { $meta: 'textScore' }
}
}
],
{
...readFromSecondary
}
)) as (DatasetDataTextSchemaType & { score: number })[];
})
);
const dataIds = Array.from(
new Set(recallResults.map((item) => item.map((item) => item.dataId)).flat())
);
const collectionIds = Array.from(
new Set(recallResults.map((item) => item.map((item) => item.collectionId)).flat())
);
// full-text 表只保存 dataId/collectionId/score，展示字段仍回查主 data 与 collection。
const [dataMaps, collectionMaps] = await Promise.all([
MongoDatasetData.find(
{
_id: { $in: dataIds }
},
datasetDataSelectField,
{ ...readFromSecondary }
)
.lean()
.then((res) => {
const map = new Map<string, DatasetDataSchemaType>();
res.forEach((item) => {
map.set(String(item._id), item);
});
return map;
}),
MongoDatasetCollection.find(
{
_id: { $in: collectionIds }
},
datasetCollectionSelectField,
{ ...readFromSecondary }
)
.lean()
.then((res) => {
const map = new Map<string, DatasetCollectionSchemaType>();
res.forEach((item) => {
map.set(String(item._id), item);
});
return map;
})
]);
const groupedRecallLists: Record<FullTextRecallSource, SearchDataResponseItemType[][]> = {
text: [],
imageCaption: []
};
recallResults.forEach((recallResult, taskIndex) => {
const task = queryTasks[taskIndex];
const list = recallResult
.map((item, index) => {
const collection = collectionMaps.get(String(item.collectionId));
if (!collection) {
logger.warn('Dataset collection not found during full-text recall', {
collectionId: item.collectionId,
dataId: item.dataId
});
return;
}
const data = dataMaps.get(String(item.dataId));
if (!data) {
logger.warn('Dataset data not found during full-text recall', {
dataId: item.dataId,
collectionId: item.collectionId
});
return;
}
return buildSearchResultItem({
data,
collection,
includeIndexes: true,
score: [
{
type: SearchScoreTypeEnum.fullText,
value: item.score || 0,
index
}
]
});
})
.filter((item) => {
if (!item) return false;
return true;
})
.map((item, index) => {
return {
...item,
score: item!.score.map((item) => ({ ...item, index }))
};
}) as SearchDataResponseItemType[];
groupedRecallLists[task.source].push(list);
});
return {
textFullTextRecallResults: concatRecallLists(groupedRecallLists.text, limit),
imageCaptionFullTextRecallResults: concatRecallLists(groupedRecallLists.imageCaption, limit)
};
};
packages/service/core/dataset/search/defaultRecall/imageCaption.ts 0 → 100644
import { getLLMModel } from '../../../ai/model';
import { createLLMResponse } from '../../../ai/llm/request';
import { getLogger, LogCategories } from '../../../../common/logger';
import { normalizeImageToBase64 } from '../utils';
import type { OpenaiAccountType } from '@fastgpt/global/support/user/team/type';
const logger = getLogger(LogCategories.MODULE.DATASET.DATA);
type ImageCaptionQueries = {
model?: string;
queries: string[];
requestIds: string[];
inputTokens: number;
outputTokens: number;
seconds: number;
usedUserOpenAIKey: boolean;
};
const emptyImageCaptionQueries = (): ImageCaptionQueries => ({
queries: [],
requestIds: [],
inputTokens: 0,
outputTokens: 0,
seconds: 0,
usedUserOpenAIKey: false
});
/**
* 将图片 query 转成可参与文本召回的图片描述 query。
* VLM 未配置、模型不支持 vision 或单张图片生成失败时都只降级图片描述召回；
* 原始图片仍可能继续走图片向量召回，所以这里不会抛出错误中断搜索。
*/
export const getImageCaptionQueries = async ({
vlmModel,
imageQueries,
userKey
}: {
vlmModel?: string;
imageQueries: string[];
userKey?: OpenaiAccountType;
}): Promise<ImageCaptionQueries> => {
if (!vlmModel || imageQueries.length === 0) {
return emptyImageCaptionQueries();
}
const vlmModelData = getLLMModel(vlmModel);
if (!vlmModelData?.vision) {
return emptyImageCaptionQueries();
}
const results = await Promise.all(
imageQueries.map(async (url, index) => {
try {
const llmStartTime = Date.now();
const {
answerText,
requestId,
usage: { inputTokens, outputTokens, usedUserOpenAIKey }
} = await createLLMResponse({
userKey,
body: {
model: vlmModelData.model,
temperature: 0.1,
stream: true,
useVision: true,
messages: [
{
role: 'user',
content: [
{
type: 'image_url',
image_url: {
url: await normalizeImageToBase64(url)
}
},
{
type: 'text',
text: '请用一句话描述这张图片的主体、场景、颜色、文字和关键视觉特征。只输出描述，不要解释。'
}
]
}
] as any
}
});
return {
query: answerText.trim(),
requestId,
inputTokens,
outputTokens,
seconds: +((Date.now() - llmStartTime) / 1000).toFixed(2),
usedUserOpenAIKey
};
} catch (error) {
logger.warn('Image caption generation failed during dataset search', {
model: vlmModelData.model,
imageIndex: index,
error
});
return {
query: '',
requestId: '',
inputTokens: 0,
outputTokens: 0,
seconds: 0,
usedUserOpenAIKey: false
};
}
})
);
const validResults = results.filter((item) => item.query);
const billableResults = results.filter((item) => item.inputTokens > 0 || item.outputTokens > 0);
return {
model: vlmModelData.model,
queries: validResults.map((item) => item.query),
requestIds: results.map((item) => item.requestId).filter(Boolean),
inputTokens: results.reduce((sum, item) => sum + item.inputTokens, 0),
outputTokens: results.reduce((sum, item) => sum + item.outputTokens, 0),
seconds: results.reduce((sum, item) => sum + item.seconds, 0),
usedUserOpenAIKey:
billableResults.length > 0 && billableResults.every((item) => item.usedUserOpenAIKey)
};
};
packages/service/core/dataset/search/defaultRecall/index.ts 0 → 100644
import {
DatasetSearchModeEnum,
DatasetSearchModeMap
} from '@fastgpt/global/core/dataset/constants';
import { addDays } from 'date-fns';
import { getDefaultRerankModel } from '../../../ai/model';
import { pushTrack } from '../../../../common/middle/tracks/utils';
import { replaceS3KeyToPreviewUrl } from '../../../../core/dataset/utils';
import type { SearchDatasetDataProps, SearchDatasetDataResponse } from '../type';
import { getImageCaptionQueries } from './imageCaption';
import { multiQueryRecall } from './multiQueryRecall';
import { reRankSearchResults } from './rerank';
import {
concatWeightedRecallLists,
filterSearchResultsByScore,
removeDuplicateSearchResults
} from './result';
import { countRecallLimit, filterDatasetDataByMaxTokens } from './utils';
/**
* 执行默认知识库召回主流程。
*
* 这条链路可以按“补充 query -> 多路召回 -> 分组融合 -> 精排/过滤 -> 输出格式化”理解：
* 1. 输入阶段：`textQueries` 是已经完成 query extension 的文本问题；`imageQueries` 是图片
* URL 或 FastGPT 内部对象 key。图片会先尝试生成 caption，把图片补成一条可搜索的文本 query。
* 2. 召回阶段：文本 query 与图片 caption 走 embedding/full-text 召回；原始图片在 embedding
* 模型支持图片时额外走图片向量召回。任一图片处理失败只丢弃该图片路径，不影响其他 query。
* 3. 融合阶段：先分别合并“用户文本”“图片 caption”“图片向量”三类结果，再按图文混合权重合成
* 最终候选集；这样文本搜索保持主导，图片结果作为视觉补充，纯图片搜索则由图片结果主导。
* 4. 精排与过滤阶段：rerank 只作用于用户文本召回，避免文本 rerank 误伤视觉相似结果；最终统一做
* 去重、相似度阈值过滤和 token 上限裁剪。
* 5. 输出阶段：只在返回前把 chunk 内容里的内部图片 key 替换为预览 URL，避免动态 URL 干扰中间去重
* 和召回融合。
*/
export async function searchDatasetData(
props: SearchDatasetDataProps
): Promise<SearchDatasetDataResponse> {
const {
teamId,
reRankQuery,
textQueries,
imageQueries = [],
userKey,
model,
vlmModel,
similarity = 0,
limit: maxTokens,
searchMode: inputSearchMode = DatasetSearchModeEnum.embedding,
embeddingWeight = 0.5,
usingReRank: inputUsingReRank = false,
rerankModel,
rerankWeight = 0.5,
datasetIds = [],
collectionFilterMatch
} = props;
const searchMode = DatasetSearchModeMap[inputSearchMode]
? inputSearchMode
: DatasetSearchModeEnum.embedding;
const usingReRank = inputUsingReRank && !!reRankQuery && !!getDefaultRerankModel();
// Step 1: 图片先尝试转成文本描述。caption 会作为普通文本 query 参与后续召回，
// 这样即使 embedding 模型不支持图片，也能通过 VLM 描述获得一条文本检索路径。
const imageCaptionQueries = await getImageCaptionQueries({
vlmModel,
imageQueries,
userKey
});
// caption 结果需要回传给工作流计费与响应详情；没有生成出有效描述时不输出该段。
const hasImageCaptionUsage =
imageCaptionQueries.inputTokens > 0 || imageCaptionQueries.outputTokens > 0;
const imageCaptionResult: SearchDatasetDataResponse['imageCaptionResult'] =
imageCaptionQueries.model && (imageCaptionQueries.queries.length > 0 || hasImageCaptionUsage)
? {
model: imageCaptionQueries.model,
inputTokens: imageCaptionQueries.inputTokens,
outputTokens: imageCaptionQueries.outputTokens,
requestIds: imageCaptionQueries.requestIds,
seconds: imageCaptionQueries.seconds,
usedUserOpenAIKey: imageCaptionQueries.usedUserOpenAIKey,
queries: imageCaptionQueries.queries
}
: undefined;
// Step 2: 根据搜索模式决定两条召回链路各自取多少候选。
// embedding/full-text/mixed 三种模式最终都走同一个 multiQueryRecall，便于统一过滤集合范围。
const { embeddingLimit, fullTextLimit } = countRecallLimit(searchMode);
const {
textEmbeddingRecallResults,
imageCaptionEmbeddingRecallResults,
imageVectorRecallResults,
textFullTextRecallResults,
imageCaptionFullTextRecallResults,
tokens: embeddingTokens
} = await multiQueryRecall({
teamId,
datasetIds,
model,
imageQueries,
collectionFilterMatch,
embeddingLimit,
fullTextLimit,
textQueries,
imageCaptionQueries: imageCaptionQueries.queries
});
// Step 3: 先在同一语义来源内融合。
// 用户文本召回由 text embedding 与 text full-text 融合；图片描述同理，
// 原始图片向量召回先独立保留，后面再和 caption 召回合并。
const textRecallResults = concatWeightedRecallLists([
{ weight: embeddingWeight, list: textEmbeddingRecallResults },
{ weight: 1 - embeddingWeight, list: textFullTextRecallResults }
]);
const imageCaptionRecallResults = concatWeightedRecallLists([
{ weight: embeddingWeight, list: imageCaptionEmbeddingRecallResults },
{ weight: 1 - embeddingWeight, list: imageCaptionFullTextRecallResults }
]);
// Step 4: rerank 只处理文本召回。
// 图片向量结果和 caption 结果仍按 RRF 融合，避免用文本 rerank 把视觉相似结果误杀。
const {
results: textRerankRecallResults,
inputTokens: reRankInputTokens,
usingReRank: finalUsingReRank
} = await reRankSearchResults({
usingReRank,
textRecallResults,
rerankModel,
query: reRankQuery,
rerankWeight
});
const hasTextQuery = textQueries.some((item) => item.trim());
// Step 5: 合并图片侧结果。caption 是图片的文本解释，权重低于原始图片向量；
// 但当模型不支持图片向量时，caption 仍能单独提供图片相关召回。
const imageRecallResults = concatWeightedRecallLists([
{
weight: imageCaptionRecallResults.length > 0 ? 0.3 : 0,
list: imageCaptionRecallResults
},
{
weight: imageVectorRecallResults.length > 0 ? 0.7 : 0,
list: imageVectorRecallResults
}
]);
// Step 6: 合并文本侧与图片侧结果。
// 纯图片搜索时图片结果权重为 1；图文混合搜索时，文本问题保持主导，图片作为补充约束。
const rrfConcatResults = concatWeightedRecallLists([
{
weight: textRerankRecallResults.length > 0 ? 1 : 0,
list: textRerankRecallResults
},
{
weight: imageRecallResults.length > 0 ? (hasTextQuery ? 0.7 : 1) : 0,
list: imageRecallResults
}
]);
// Step 7: 最终过滤顺序固定为：同内容去重 -> 相似度阈值 -> token 上限。
// 先去重可以避免同一 chunk 因多路召回重复占用相似度过滤和 token 预算。
const filterSameDataResults = removeDuplicateSearchResults(rrfConcatResults);
const { results: scoreFilter, usingSimilarityFilter } = filterSearchResultsByScore({
data: filterSameDataResults,
usingReRank: finalUsingReRank,
searchMode,
similarity
});
const filterMaxTokensResult = await filterDatasetDataByMaxTokens(scoreFilter, maxTokens);
// Step 8: 返回前把 q 中的内部图片 key 转为可预览 URL。
// 只在最终结果处理，避免中间召回和去重阶段混入带过期时间的动态 URL。
const finalResult = filterMaxTokensResult.map((item) => {
item.q = replaceS3KeyToPreviewUrl(item.q, addDays(new Date(), 90));
return item;
});
pushTrack.datasetSearch({ datasetIds, teamId });
return {
searchRes: finalResult,
embeddingTokens,
reRankInputTokens,
searchMode,
limit: maxTokens,
similarity,
usingReRank: finalUsingReRank,
usingSimilarityFilter,
imageCaptionResult
};
}
packages/service/core/dataset/search/defaultRecall/multiQueryRecall.ts 0 → 100644
import { getForbidCollectionIdList, filterCollectionByMetadata } from './collectionFilter';
import { embeddingRecall } from './embeddingRecall';
import { fullTextRecall } from './fullTextRecall';
/**
* 默认召回的并行调度层。
* 这里先统一计算 forbid collection 与 metadata filter，再把同一份 collection 约束
* 下发给 embedding/full-text 两条召回链路，保证两种召回方式看到的集合范围一致。
*/
export const multiQueryRecall = async ({
teamId,
datasetIds,
model,
imageQueries,
collectionFilterMatch,
embeddingLimit,
fullTextLimit,
textQueries,
imageCaptionQueries
}: {
teamId: string;
datasetIds: string[];
model: string;
imageQueries: string[];
collectionFilterMatch?: string;
embeddingLimit: number;
fullTextLimit: number;
textQueries: string[];
imageCaptionQueries: string[];
}) => {
const [forbidCollectionIdList, filterCollectionIdList] = await Promise.all([
getForbidCollectionIdList({
teamId,
datasetIds
}),
filterCollectionByMetadata({
teamId,
datasetIds,
collectionFilterMatch
})
]);
const [
{
tokens,
textEmbeddingRecallResults,
imageCaptionEmbeddingRecallResults,
imageVectorRecallResults
},
{ textFullTextRecallResults, imageCaptionFullTextRecallResults }
] = await Promise.all([
embeddingRecall({
teamId,
datasetIds,
model,
imageQueries,
textQueries,
imageCaptionQueries,
limit: embeddingLimit,
forbidCollectionIdList,
filterCollectionIdList
}),
fullTextRecall({
teamId,
datasetIds,
queryGroups: [
{ source: 'text', queries: textQueries },
{ source: 'imageCaption', queries: imageCaptionQueries }
],
limit: fullTextLimit,
filterCollectionIdList,
forbidCollectionIdList
})
]);
return {
tokens,
textEmbeddingRecallResults,
imageCaptionEmbeddingRecallResults,
imageVectorRecallResults,
textFullTextRecallResults,
imageCaptionFullTextRecallResults
};
};
packages/service/core/dataset/search/defaultRecall/rerank.ts 0 → 100644
import { SearchScoreTypeEnum } from '@fastgpt/global/core/dataset/constants';
import type { RerankModelItemType } from '@fastgpt/global/core/ai/model.schema';
import type { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
import { reRankRecall } from '../../../../core/ai/rerank';
import { concatWeightedRecallLists, removeDuplicateSearchResults } from './result';
const datasetDataReRank = async ({
rerankModel,
data,
query
}: {
rerankModel?: RerankModelItemType;
data: SearchDataResponseItemType[];
query: string;
}): Promise<{
results: SearchDataResponseItemType[];
inputTokens: number;
}> => {
const { results, inputTokens } = await reRankRecall({
model: rerankModel,
query,
documents: data.map((item) => ({
id: item.id,
text: `${item.q}\n${item.a}`.trim()
}))
});
if (results.length === 0) {
return Promise.reject('Rerank error');
}
// add new score to data
const mergeResult = results
.map((item, index) => {
const target = data.find((dataItem) => dataItem.id === item.id);
if (!target) return null;
const score = item.score || 0;
return {
...target,
score: [{ type: SearchScoreTypeEnum.reRank, value: score, index }]
};
})
.filter(Boolean) as SearchDataResponseItemType[];
return {
results: mergeResult,
inputTokens
};
};
/**
* 只对文本召回结果 rerank。图片召回仍通过 RRF 权重参与最终融合，避免图片向量结果被文本 rerank 误杀。
*/
export const reRankSearchResults = async ({
usingReRank,
textRecallResults,
rerankModel,
query,
rerankWeight
}: {
usingReRank: boolean;
textRecallResults: SearchDataResponseItemType[];
rerankModel?: RerankModelItemType;
query: string;
rerankWeight: number;
}): Promise<{
results: SearchDataResponseItemType[];
inputTokens: number;
usingReRank: boolean;
}> => {
if (!usingReRank || !query || textRecallResults.length === 0) {
return {
results: textRecallResults,
inputTokens: 0,
usingReRank: false
};
}
try {
const { results: reRankResults, inputTokens } = await datasetDataReRank({
rerankModel,
query,
data: removeDuplicateSearchResults(textRecallResults)
});
if (rerankWeight === 1) {
return {
results: reRankResults,
inputTokens,
usingReRank: true
};
}
return {
results: concatWeightedRecallLists([
{ weight: 1 - rerankWeight, list: textRecallResults },
{ weight: rerankWeight, list: reRankResults }
]),
inputTokens,
usingReRank: true
};
} catch {
return {
results: textRecallResults,
inputTokens: 0,
usingReRank: false
};
}
};
packages/service/core/dataset/search/defaultRecall/result.ts 0 → 100644
import { hashStr } from '@fastgpt/global/common/string/tools';
import { DatasetSearchModeEnum, SearchScoreTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { getCollectionSourceData } from '@fastgpt/global/core/dataset/collection/utils';
import { datasetSearchResultConcat } from '@fastgpt/global/core/dataset/search/utils';
import type {
DatasetCollectionSchemaType,
DatasetDataSchemaType,
SearchDataResponseItemType
} from '@fastgpt/global/core/dataset/type';
import { formatDatasetDataValue } from '../../data/controller';
/**
* 把召回命中的 data 与 collection 统一整理成搜索结果。
* embedding/full-text 召回只负责生成各自的 score，展示字段和来源字段在这里保持一致。
*/
export const buildSearchResultItem = ({
data,
collection,
score,
includeIndexes = false
}: {
data: DatasetDataSchemaType;
collection: DatasetCollectionSchemaType;
score: SearchDataResponseItemType['score'];
includeIndexes?: boolean;
}): SearchDataResponseItemType => ({
id: String(data._id),
updateTime: data.updateTime,
...formatDatasetDataValue({
q: data.q,
a: data.a,
imageId: data.imageId,
imageDescMap: data.imageDescMap
}),
chunkIndex: data.chunkIndex,
...(includeIndexes ? { indexes: data.indexes } : {}),
datasetId: String(data.datasetId),
collectionId: String(data.collectionId),
...getCollectionSourceData(collection),
score
});
export const concatRecallLists = (lists: SearchDataResponseItemType[][], limit: number) => {
return datasetSearchResultConcat(lists.map((list) => ({ weight: 1, list }))).slice(0, limit);
};
export const concatWeightedRecallLists = (
lists: { weight: number; list: SearchDataResponseItemType[] }[]
) => {
return datasetSearchResultConcat(lists.filter((item) => item.weight > 0 && item.list.length > 0));
};
/**
* 按 q+a 内容去重。召回链路里同一文本块可能同时被文本、图片描述、图片向量命中，
* 这里用归一化后的内容 hash 保留最前面的排序结果。
*/
export const removeDuplicateSearchResults = (data: SearchDataResponseItemType[]) => {
const set = new Set<string>();
return data.filter((item) => {
// 删除所有的标点符号与空格等，只对文本进行比较
const str = hashStr(`${item.q}${item.a}`.replace(/[^\p{L}\p{N}]/gu, ''));
if (set.has(str)) return false;
set.add(str);
return true;
});
};
export const filterSearchResultsByScore = ({
data,
usingReRank,
searchMode,
similarity
}: {
data: SearchDataResponseItemType[];
usingReRank: boolean;
searchMode: DatasetSearchModeEnum;
similarity: number;
}) => {
const scoreType = usingReRank
? SearchScoreTypeEnum.reRank
: searchMode === DatasetSearchModeEnum.embedding
? SearchScoreTypeEnum.embedding
: undefined;
if (!scoreType) {
return {
results: data,
usingSimilarityFilter: false
};
}
return {
results: data.filter((item) => {
const targetScore = item.score.find((item) => item.type === scoreType);
return !targetScore || targetScore.value >= similarity;
}),
usingSimilarityFilter: true
};
};
packages/service/core/dataset/search/defaultRecall/utils.ts 0 → 100644
import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constants';
import type { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
import { countPromptTokens } from '../../../../common/string/tiktoken/index';
/**
* 根据搜索模式分配每条召回链路的候选数量。
* mixed 模式会让 embedding 与 full-text 都多取一批候选，再交给 RRF/权重融合；
* 单一路径模式则关闭另一条召回，避免无意义查询。
*/
export const countRecallLimit = (searchMode: DatasetSearchModeEnum) => {
if (searchMode === DatasetSearchModeEnum.embedding) {
return {
embeddingLimit: 100,
fullTextLimit: 0
};
}
if (searchMode === DatasetSearchModeEnum.fullTextRecall) {
return {
embeddingLimit: 0,
fullTextLimit: 100
};
}
return {
embeddingLimit: 80,
fullTextLimit: 60
};
};
/**
* 按模型上下文 token 上限截断最终引用列表。
* 至少保留第一条结果：否则高质量但单条超长的命中会被全部过滤，导致用户看到
* “无引用”，这比返回一条可裁剪引用更难排查。
*/
export const filterDatasetDataByMaxTokens = async (
data: SearchDataResponseItemType[],
maxTokens: number
) => {
const tokensScoreFilter = await Promise.all(
data.map(async (item) => ({
...item,
tokens: await countPromptTokens(item.q + item.a)
}))
);
const results: SearchDataResponseItemType[] = [];
let totalTokens = 0;
for await (const item of tokensScoreFilter) {
results.push(item);
totalTokens += item.tokens;
if (totalTokens > maxTokens) {
break;
}
}
return results.length === 0 ? data.slice(0, 1) : results;
};
packages/service/core/dataset/search/index.ts 0 → 100644
import type {
DeepRagSearchProps,
DefaultSearchDatasetDataProps,
SearchDatasetDataResponse
} from './type';
import { getLLMModel } from '../../ai/model';
import { searchDatasetData } from './defaultRecall';
import { datasetSearchQueryExtension } from './utils';
export * from './type';
/**
* 知识库搜索统一入口。
*
* 入口层负责搜索前置编排：文本 query extension、rerank query 生成，以及后续搜索方式分发。
* 当前只接入默认召回实现，调用方不直接依赖具体 recall 目录。
*/
export const defaultSearchDatasetData = async ({
datasetSearchUsingExtensionQuery,
datasetSearchExtensionModel,
datasetSearchExtensionBg,
userKey,
...props
}: DefaultSearchDatasetDataProps): Promise<SearchDatasetDataResponse> => {
const textQueries = props.textQueries.map((query) => query.trim()).filter(Boolean);
const query = textQueries.join('\n');
const { searchQueries, reRankQuery, aiExtensionResult } = query
? await datasetSearchQueryExtension({
query,
llmModel: datasetSearchUsingExtensionQuery
? getLLMModel(datasetSearchExtensionModel)?.model
: undefined,
embeddingModel: props.model,
extensionBg: datasetSearchExtensionBg,
histories: props.histories,
userKey
})
: {
searchQueries: [],
reRankQuery: query,
aiExtensionResult: undefined
};
const result = await searchDatasetData({
...props,
userKey,
reRankQuery,
textQueries: searchQueries
});
return {
...result,
queryExtensionResult: aiExtensionResult
? {
llmModel: aiExtensionResult.llmModel,
requestId: aiExtensionResult.requestId,
seconds: aiExtensionResult.seconds,
inputTokens: aiExtensionResult.inputTokens,
outputTokens: aiExtensionResult.outputTokens,
usedUserOpenAIKey: aiExtensionResult.usedUserOpenAIKey,
embeddingModel: aiExtensionResult.embeddingModel,
embeddingTokens: aiExtensionResult.embeddingTokens,
query: searchQueries.join('\n')
}
: undefined
};
};
/**
* Deep RAG 搜索入口由搜索分发层统一暴露，避免具体 recall 实现持有全局 handler 适配。
*/
export const deepRagSearch = (data: DeepRagSearchProps): Promise<SearchDatasetDataResponse> =>
global.deepRagHandler(data);
packages/service/core/dataset/search/type.ts 0 → 100644
import type { RerankModelItemType } from '@fastgpt/global/core/ai/model.schema';
import type { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constants';
import type { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
import type { ChatItemMiniType } from '@fastgpt/global/core/chat/type';
import type { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import type { OpenaiAccountType } from '@fastgpt/global/support/user/team/type';
export type SearchDatasetDataProps = {
histories: ChatItemMiniType[];
teamId: string;
uid?: string;
tmbId?: string;
model: string;
vlmModel?: string;
datasetIds: string[];
reRankQuery: string;
// 工作流入口归一化后的文本 query。
textQueries: string[];
// 工作流入口归一化后的图片 query。
imageQueries?: string[];
// 外部 OpenAI 账号。默认召回里的辅助 LLM 请求需要沿用它来保持计费一致。
userKey?: OpenaiAccountType;
[NodeInputKeyEnum.datasetSimilarity]?: number; // min distance
[NodeInputKeyEnum.datasetMaxTokens]: number; // max Token limit
[NodeInputKeyEnum.datasetSearchMode]?: DatasetSearchModeEnum;
[NodeInputKeyEnum.datasetSearchEmbeddingWeight]?: number;
[NodeInputKeyEnum.datasetSearchUsingReRank]?: boolean;
[NodeInputKeyEnum.datasetSearchRerankModel]?: RerankModelItemType;
[NodeInputKeyEnum.datasetSearchRerankWeight]?: number;
/*
{
tags: {
$and: ["str1","str2"],
$or: ["str1","str2",null] null means no tags
},
createTime: {
$gte: 'xx',
$lte: 'xxx'
}
}
*/
collectionFilterMatch?: string;
};
export type SearchDatasetDataResponse = {
searchRes: SearchDataResponseItemType[];
embeddingTokens: number;
reRankInputTokens: number;
searchMode: `${DatasetSearchModeEnum}`;
limit: number;
similarity: number;
usingReRank: boolean;
usingSimilarityFilter: boolean;
queryExtensionResult?: {
llmModel: string;
embeddingModel: string;
requestId: string;
seconds: number;
inputTokens: number;
outputTokens: number;
usedUserOpenAIKey: boolean;
embeddingTokens: number;
query: string;
};
deepSearchResult?: { model: string; inputTokens: number; outputTokens: number };
imageCaptionResult?: {
model: string;
inputTokens: number;
outputTokens: number;
requestIds: string[];
seconds: number;
usedUserOpenAIKey: boolean;
queries: string[];
};
};
export type DefaultSearchDatasetDataProps = Omit<SearchDatasetDataProps, 'reRankQuery'> & {
[NodeInputKeyEnum.datasetSearchUsingExtensionQuery]?: boolean;
[NodeInputKeyEnum.datasetSearchExtensionModel]?: string;
[NodeInputKeyEnum.datasetSearchExtensionBg]?: string;
userKey?: OpenaiAccountType;
};
export type DeepRagSearchProps = Omit<SearchDatasetDataProps, 'reRankQuery'> & {
[NodeInputKeyEnum.datasetDeepSearchModel]?: string;
[NodeInputKeyEnum.datasetDeepSearchMaxTimes]?: number;
[NodeInputKeyEnum.datasetDeepSearchBg]?: string;
};
packages/service/core/dataset/search/utils.ts
......@@ -3,9 +3,16 @@ import { type ChatItemMiniType } from '@fastgpt/global/core/chat/type';
import { hashStr } from '@fastgpt/global/common/string/tools';
import { getLogger, LogCategories } from '../../../common/logger';
import type { OpenaiAccountType } from '@fastgpt/global/support/user/team/type';
import { getImageBase64 } from '../../../common/file/image/utils';
import { serviceEnv } from '../../../env';
const logger = getLogger(LogCategories.MODULE.DATASET.DATA);
/**
* 计算多个 collection 过滤条件的交集。
* `undefined` 表示当前过滤维度未启用，应被忽略；空数组表示该维度明确无命中，
* 会参与交集并让最终结果为空。
*/
export const computeFilterIntersection = (lists: (string[] | undefined)[]) => {
const validLists = lists.filter((list): list is string[] => list !== undefined);
......@@ -18,6 +25,33 @@ export const computeFilterIntersection = (lists: (string[] | undefined)[]) => {
});
};
/**
* 按环境开关规范化图片输入。
* data URL 已经是模型可读内容，始终原样返回；普通图片 URL 只有
* serviceEnv.MULTIPLE_DATA_TO_BASE64 为 true 时才转成 base64。
* FastGPT 内部对象 key 的鉴权和临时 URL 生成应在入口层完成，避免通用规范化函数
* 混入业务权限和存储来源判断。
* 这里不吞异常，由上层按图片粒度降级，避免一张坏图中断整次检索。
*/
export const normalizeImageToBase64 = async (imageUrl: string) => {
if (imageUrl.startsWith('data:image/')) {
return imageUrl;
}
if (!serviceEnv.MULTIPLE_DATA_TO_BASE64) {
return imageUrl;
}
const { completeBase64 } = await getImageBase64(imageUrl);
return completeBase64;
};
/**
* 对文本查询做 query extension。
* 调用方会先把多个文本 query 合并成一个字符串传入，这里始终按普通字符串处理，
* 不再兼容旧的“query 已经是扩展结果 JSON”分支。扩展失败时返回原始 query，
* 保证搜索主链路不被 LLM 扩展能力影响。
*/
export const datasetSearchQueryExtension = async ({
query,
llmModel,
......@@ -33,49 +67,32 @@ export const datasetSearchQueryExtension = async ({
extensionBg?: string;
histories?: ChatItemMiniType[];
}) => {
const filterSamQuery = (queries: string[]) => {
/**
* query extension 结果可能与原 query 只有标点或空格差异。
* 去重时忽略标点和空白，但保留原始文本，避免影响后续 embedding 和展示。
*/
const filterSameQuery = (queries: string[]) => {
const set = new Set<string>();
const filterSameQueries = queries.filter((item) => {
// 删除所有的标点符号与空格等，只对文本进行比较
const str = hashStr(item.replace(/[^\p{L}\p{N}]/gu, ''));
if (set.has(str)) return false;
set.add(str);
return true;
});
const filterSameQueries = queries
.map((item) => item.trim())
.filter(Boolean)
.filter((item) => {
// 删除所有的标点符号与空格等，只对文本进行比较
const str = hashStr(item.replace(/[^\p{L}\p{N}]/gu, ''));
if (set.has(str)) return false;
set.add(str);
return true;
});
return filterSameQueries;
};
// 检查传入的 query 是否已经进行过扩展
const {
queries: initQueries,
reRankQuery: initReRankQuery,
alreadyExtension
} = (() => {
/* if query already extension, direct parse */
try {
const jsonParse = JSON.parse(query);
const queries: string[] = Array.isArray(jsonParse) ? filterSamQuery(jsonParse) : [query];
const alreadyExtension = Array.isArray(jsonParse);
return {
queries,
reRankQuery: alreadyExtension ? queries.join('\n') : query,
alreadyExtension
};
} catch {
return {
queries: [query],
reRankQuery: query,
alreadyExtension: false
};
}
})();
let queries = initQueries;
let reRankQuery = initReRankQuery;
let queries = [query];
let reRankQuery = query;
// Use LLM to generate extension queries
const aiExtensionResult = await (async () => {
if (!llmModel || !embeddingModel || alreadyExtension) return;
if (!llmModel || !embeddingModel) return;
try {
const result = await queryExtension({
......@@ -94,7 +111,7 @@ export const datasetSearchQueryExtension = async ({
})();
if (aiExtensionResult) {
queries = queries.concat(aiExtensionResult.extensionQueries);
queries = filterSameQuery(queries.concat(aiExtensionResult.extensionQueries));
reRankQuery = queries.join('\n');
}
......
packages/service/core/workflow/dispatch/ai/agent/sub/dataset/index.ts
......@@ -12,7 +12,7 @@ import { MongoDataset } from '../../../../../../dataset/schema';
import {
defaultSearchDatasetData,
type DefaultSearchDatasetDataProps
} from '../../../../../../dataset/search/controller';
} from '../../../../../../dataset/search';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { getLogger, LogCategories } from '../../../../../../../common/logger';
import type { DispatchSubAppResponse } from '../../type';
......@@ -199,8 +199,7 @@ export const dispatchAgentDatasetSearch = async ({
const searchData: DefaultSearchDatasetDataProps = {
histories: [],
teamId,
reRankQuery: query,
queries: [query],
textQueries: [query],
model: vectorModel.model,
similarity: datasetParams.similarity ?? 0.4,
limit: datasetParams.limit || 5000,
......
packages/service/core/workflow/dispatch/ai/toolcall/hooks/useToolRunner.ts
import type { ChatCompletionMessageToolCall } from '@fastgpt/global/core/ai/llm/type';
import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants';
import { chats2GPTMessages } from '@fastgpt/global/core/chat/adapt';
import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
import type { RuntimeNodeItemType } from '@fastgpt/global/core/workflow/runtime/type';
import type { WorkflowInteractiveResponseType } from '@fastgpt/global/core/workflow/template/system/interactive/type';
import type { AgentLoopChildrenInteractiveParams } from '../../../../../ai/llm/agentLoop';
import { runSandboxTools } from '../../../../../ai/sandbox/toolCall';
......@@ -43,11 +46,42 @@ const getAssistantMessages = (assistantResponses: DispatchFlowResponse['assistan
reserveId: false
});
/**
* ToolCall 调用知识库搜索节点时，模型只生成文本 query；父节点输入文件需要在这里追加。
* 这里只做最小拼接，不判断文件类型：搜索节点会统一调用 normalizeDatasetSearchInput
* 拆分文本和图片。清空 userChatInput 是为了让搜索节点走 datasetSearchInput 数组，
* 避免旧字段优先级导致追加的文件 URL 被忽略。
*/
const mergeDatasetToolFileUrls = ({
flowNodeType,
startParams,
fileUrls = []
}: {
flowNodeType: RuntimeNodeItemType['flowNodeType'];
startParams: Record<string, any>;
fileUrls?: string[];
}) => {
if (flowNodeType !== FlowNodeTypeEnum.datasetSearchNode || fileUrls.length === 0) {
return startParams;
}
const queryInput =
startParams[NodeInputKeyEnum.datasetSearchInput] ?? startParams[NodeInputKeyEnum.userChatInput];
const queryList = Array.isArray(queryInput) ? queryInput : queryInput ? [queryInput] : [];
return {
...startParams,
[NodeInputKeyEnum.userChatInput]: '',
[NodeInputKeyEnum.datasetSearchInput]: [...queryList, ...fileUrls]
};
};
export const useToolRunner = ({
workflowProps,
runtimeNodes,
runtimeEdges,
allFiles,
fileUrls = [],
getToolInfo,
cacheToolFlowResponse,
appendToolFlowResponse,
......@@ -57,6 +91,7 @@ export const useToolRunner = ({
runtimeNodes: DispatchToolModuleProps['runtimeNodes'];
runtimeEdges: DispatchToolModuleProps['runtimeEdges'];
allFiles: Map<string, FileInputType>;
fileUrls?: string[];
getToolInfo: (name: string) => ToolInfo | undefined;
cacheToolFlowResponse: (args: {
call: ChatCompletionMessageToolCall;
......@@ -133,7 +168,11 @@ export const useToolRunner = ({
* 用户配置的工具节点会在当前 runtime 副本中被标记为入口节点；
* 参数只注入入口节点，后续依旧走原 workflow 的边和节点调度。
*/
const startParams = parseJsonArgs(call.function.arguments);
const startParams = mergeDatasetToolFileUrls({
flowNodeType: toolNode.flowNodeType,
startParams: parseJsonArgs(call.function.arguments) ?? {},
fileUrls
});
initToolNodes(runtimeNodes, [toolNode.nodeId], startParams);
initToolCallEdges(runtimeEdges, [toolNode.nodeId]);
......
packages/service/core/workflow/dispatch/ai/toolcall/index.ts
import { NodeInputKeyEnum, NodeOutputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import { NodeOutputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants';
import type { DispatchNodeResultType } from '@fastgpt/global/core/workflow/runtime/type';
import { getLLMModel } from '../../../../ai/model';
......@@ -18,7 +18,7 @@ type Response = DispatchNodeResultType<{
export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<Response> => {
const {
node: { nodeId, isEntry, inputs },
node: { nodeId, isEntry },
runtimeNodes,
runtimeEdges,
histories,
......@@ -33,14 +33,13 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
systemPrompt,
userChatInput,
history = 6,
fileUrlList,
fileUrlList: fileLinks,
aiChatVision,
aiChatReasoning,
isResponseAnswerText = true,
useAgentSandbox
}
} = props;
let fileLinks = fileUrlList;
const useSandbox = !!useAgentSandbox && !!global.feConfigs?.show_agent_sandbox;
......@@ -51,10 +50,6 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
props.params.aiChatVision = aiChatVision && toolModel.vision;
props.params.aiChatReasoning = aiChatReasoning && toolModel.reasoning;
const fileUrlInput = inputs.find((item) => item.key === NodeInputKeyEnum.fileUrlList);
if (!fileUrlInput || !fileUrlInput.value || fileUrlInput.value.length === 0) {
fileLinks = undefined;
}
const toolNodes = useToolNodeList({
nodeId,
......
packages/service/core/workflow/dispatch/ai/toolcall/toolCall.ts
......@@ -36,7 +36,6 @@ export const runToolCall = async (props: DispatchToolModuleProps): Promise<Respo
childrenInteractiveParams,
allFiles,
currentInputFiles,
...workflowProps
} = props;
const {
......@@ -60,7 +59,8 @@ export const runToolCall = async (props: DispatchToolModuleProps): Promise<Respo
aiChatReasoning,
aiChatReasoningEffort,
isResponseAnswerText = true,
useAgentSandbox
useAgentSandbox,
fileUrlList
}
} = workflowProps;
......@@ -98,6 +98,7 @@ export const runToolCall = async (props: DispatchToolModuleProps): Promise<Respo
runtimeNodes,
runtimeEdges,
allFiles,
fileUrls: fileUrlList,
getToolInfo,
cacheToolFlowResponse,
appendToolFlowResponse,
......
packages/service/core/workflow/dispatch/ai/toolcall/type.ts
......@@ -35,6 +35,7 @@ export type DispatchToolModuleProps = ModuleDispatchProps<{
childrenInteractiveParams?: AgentLoopChildrenInteractiveParams<WorkflowInteractiveResponseType>;
allFiles: Map<string, FileInputType>;
currentInputFiles: FileInputType[];
fileUrls?: string[];
};
export type ToolNodeItemType = {
......
packages/service/core/workflow/dispatch/dataset/nodeResponse.ts
......@@ -8,17 +8,20 @@ const datasetSearchModuleLogo = 'core/workflow/template/datasetSearch';
const createDatasetSearchChildNodeResponse = ({
requestId,
requestIds,
usage,
moduleName,
seconds,
textOutput
}: {
requestId?: string;
requestIds?: string[];
usage: ChatNodeUsageType;
moduleName: string;
seconds?: number;
textOutput?: string;
}): ChatHistoryItemResType => {
const llmRequestIds = requestIds?.length ? requestIds : requestId ? [requestId] : undefined;
const id = requestId || getNanoid();
return {
......@@ -29,7 +32,7 @@ const createDatasetSearchChildNodeResponse = ({
moduleLogo: datasetSearchModuleLogo,
runningTime: seconds,
model: usage.model,
llmRequestIds: requestId ? [requestId] : undefined,
llmRequestIds,
inputTokens: usage.inputTokens,
outputTokens: usage.outputTokens,
totalPoints: usage.totalPoints,
......@@ -61,6 +64,29 @@ export const createQueryExtensionChildNodeResponse = ({
});
/**
* 创建知识库搜索里的图片解析子 nodeResponse。
* 一次图搜可能解析多张图片，所以这里保留所有 caption LLM requestId。
*/
export const createImageCaptionChildNodeResponse = ({
requestIds,
usage,
seconds,
queries
}: {
requestIds?: string[];
usage: ChatNodeUsageType;
seconds?: number;
queries: string[];
}) =>
createDatasetSearchChildNodeResponse({
requestIds,
usage,
seconds,
moduleName: i18nT('account_usage:image_parse'),
textOutput: queries.join('\n')
});
/**
* 创建知识库搜索里的 chunk selection 子 nodeResponse。
* Agent 知识库工具会在搜索结果过长时触发该 LLM 裁切请求，requestId 只挂在这个子节点上。
*/
......
packages/service/core/workflow/dispatch/dataset/search.ts
......@@ -4,7 +4,7 @@ import type { SelectedDatasetType } from '@fastgpt/global/core/workflow/type/io'
import type { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
import type { ModuleDispatchProps } from '@fastgpt/global/core/workflow/runtime/type';
import { getEmbeddingModel, getRerankModel } from '../../../ai/model';
import { deepRagSearch, defaultSearchDatasetData } from '../../../dataset/search/controller';
import { deepRagSearch, defaultSearchDatasetData } from '../../../dataset/search';
import type { NodeInputKeyEnum, NodeOutputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants';
import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constants';
......@@ -16,7 +16,11 @@ import { getDatasetSearchToolResponsePrompt } from '@fastgpt/global/core/ai/prom
import { getNodeErrResponse } from '../utils';
import { getLogger, LogCategories } from '../../../../common/logger';
import type { ChatHistoryItemResType } from '@fastgpt/global/core/chat/type';
import { createQueryExtensionChildNodeResponse } from './nodeResponse';
import {
createImageCaptionChildNodeResponse,
createQueryExtensionChildNodeResponse
} from './nodeResponse';
import { normalizeDatasetSearchInput } from './utils';
const logger = getLogger(LogCategories.MODULE.WORKFLOW.DATASET);
......@@ -25,6 +29,7 @@ type DatasetSearchProps = ModuleDispatchProps<{
[NodeInputKeyEnum.datasetSimilarity]: number;
[NodeInputKeyEnum.datasetMaxTokens]: number;
[NodeInputKeyEnum.userChatInput]?: string;
[NodeInputKeyEnum.datasetSearchInput]?: string[];
[NodeInputKeyEnum.datasetSearchMode]: DatasetSearchModeEnum;
[NodeInputKeyEnum.datasetSearchEmbeddingWeight]?: number;
......@@ -62,6 +67,7 @@ export async function dispatchDatasetSearch(
similarity,
limit = 5000,
userChatInput = '',
datasetSearchInput = [],
authTmbId = false,
collectionFilterMatch,
searchMode,
......@@ -102,7 +108,10 @@ export async function dispatchDatasetSearch(
[DispatchNodeResponseKeyEnum.toolResponses]: []
};
if (!userChatInput) {
const searchQueries = userChatInput ? [userChatInput] : datasetSearchInput;
const { textQueries, imageQueries } = normalizeDatasetSearchInput(searchQueries);
if (textQueries.length === 0 && imageQueries.length === 0) {
return emptyResult;
}
......@@ -119,9 +128,11 @@ export async function dispatchDatasetSearch(
}
// Get vector model
const vectorModel = getEmbeddingModel(
(await MongoDataset.findById(datasets[0].datasetId, 'vectorModel').lean())?.vectorModel
);
const dataset = await MongoDataset.findById(
datasets[0].datasetId,
'vectorModel vlmModel'
).lean();
const vectorModel = getEmbeddingModel(dataset?.vectorModel);
// Get Rerank Model
const rerankModelData = getRerankModel(rerankModel);
......@@ -129,9 +140,10 @@ export async function dispatchDatasetSearch(
const searchData = {
histories,
teamId,
reRankQuery: userChatInput,
queries: [userChatInput],
textQueries,
imageQueries,
model: vectorModel.model,
vlmModel: dataset?.vlmModel,
similarity,
limit,
datasetIds,
......@@ -142,6 +154,7 @@ export async function dispatchDatasetSearch(
rerankWeight,
collectionFilterMatch
};
const useDeepSearch = datasetDeepSearch && textQueries.length > 0;
const {
searchRes,
embeddingTokens,
......@@ -149,8 +162,9 @@ export async function dispatchDatasetSearch(
usingSimilarityFilter,
usingReRank: searchUsingReRank,
queryExtensionResult,
imageCaptionResult,
deepSearchResult
} = datasetDeepSearch
} = useDeepSearch
? await deepRagSearch({
...searchData,
datasetDeepSearchModel,
......@@ -233,7 +247,32 @@ export async function dispatchDatasetSearch(
outputTokens: 0
});
}
// 4. Deep search
// 4. Image caption
if (imageCaptionResult) {
const { totalPoints, modelName } = formatModelChars2Points({
model: imageCaptionResult.model,
inputTokens: imageCaptionResult.inputTokens,
outputTokens: imageCaptionResult.outputTokens
});
const imageCaptionPoints = imageCaptionResult.usedUserOpenAIKey ? 0 : totalPoints;
const imageCaptionUsage: ChatNodeUsageType = {
totalPoints: imageCaptionPoints,
moduleName: i18nT('account_usage:image_parse'),
model: modelName,
inputTokens: imageCaptionResult.inputTokens,
outputTokens: imageCaptionResult.outputTokens
};
nodeUsages.push(imageCaptionUsage);
childrenResponses.push(
createImageCaptionChildNodeResponse({
requestIds: imageCaptionResult.requestIds,
usage: imageCaptionUsage,
seconds: imageCaptionResult.seconds,
queries: imageCaptionResult.queries
})
);
}
// 5. Deep search
if (deepSearchResult) {
const { totalPoints, modelName } = formatModelChars2Points({
model: deepSearchResult.model,
......@@ -262,7 +301,7 @@ export async function dispatchDatasetSearch(
},
[DispatchNodeResponseKeyEnum.nodeResponse]: {
totalPoints,
query: userChatInput,
datasetQueries: [...textQueries, ...imageQueries],
embeddingModel: vectorModel.name,
embeddingTokens,
similarity: usingSimilarityFilter ? similarity : undefined,
......
packages/service/core/workflow/dispatch/dataset/utils.ts 0 → 100644
import { ChatFileTypeEnum } from '@fastgpt/global/core/chat/constants';
import { parseUrlToFileType } from '../../utils/context';
export type NormalizeDatasetSearchInputResult = {
textQueries: string[];
imageQueries: string[];
};
const httpUrlReg = /^https?:\/\//i;
const pushUnique = <T>(list: T[], seen: Set<T>, value: T) => {
if (!seen.has(value)) {
seen.add(value);
list.push(value);
}
};
const isHttpUrl = (input: string) => httpUrlReg.test(input);
/**
* 将数据集搜索输入拆成文本查询和图片查询。
* datasetSearchInput 会同时接收用户问题和 userFiles；这里只把普通 http(s) URL
* 作为文件候选继续判断，非 http(s) 输入都保留为文本检索 query。
*/
export const normalizeDatasetSearchInput = (
inputList: string[]
): NormalizeDatasetSearchInputResult => {
const textQueries: string[] = [];
const imageQueries: string[] = [];
const seenTextQueries = new Set<string>();
const seenQueryImageUrls = new Set<string>();
for (const rawInput of inputList) {
const input = rawInput.trim();
if (!input) continue;
if (!isHttpUrl(input)) {
pushUnique(textQueries, seenTextQueries, input);
continue;
}
const fileInfo = parseUrlToFileType(input);
if (fileInfo?.type !== ChatFileTypeEnum.image || seenQueryImageUrls.has(input)) {
continue;
}
seenQueryImageUrls.add(input);
imageQueries.push(input);
}
return {
textQueries,
imageQueries
};
};
packages/service/core/workflow/utils/context.ts
......@@ -102,7 +102,7 @@ export const parseUrlToFileType = (url: string): UserChatItemFileItemType | unde
name: filename ? decodeURIComponent(filename) : url,
url
};
} catch (error) {
} catch {
return {
type: ChatFileTypeEnum.file,
name: url,
......
packages/service/test/common/vectorDB/controller.test.ts
......@@ -5,11 +5,10 @@ import {
mockVectorEmbRecall,
mockVectorInit,
mockGetVectorDataByTime,
mockGetVectorCountByTeamId,
mockGetVectorCount,
resetVectorMocks
} from '@test/mocks/common/vector';
import { mockGetVectorsByText } from '@test/mocks/core/ai/embedding';
import { mockGetVectors } from '@test/mocks/core/ai/embedding';
// Import controller functions after mocks are set up
import {
......@@ -65,7 +64,7 @@ describe('VectorDB Controller', () => {
mockSetRedisCache.mockReset();
mockDelRedisCache.mockReset();
mockLoggerWarn.mockReset();
mockGetVectorsByText.mockClear();
mockGetVectors.mockClear();
});
afterEach(() => {
......@@ -248,7 +247,7 @@ describe('VectorDB Controller', () => {
[0.1, 0.2],
[0.3, 0.4]
];
mockGetVectorsByText.mockResolvedValue({
mockGetVectors.mockResolvedValue({
tokens: 100,
vectors: mockVectors
});
......@@ -264,9 +263,18 @@ describe('VectorDB Controller', () => {
model: mockModel as any
});
expect(mockGetVectorsByText).toHaveBeenCalledWith({
expect(mockGetVectors).toHaveBeenCalledWith({
model: mockModel,
input: ['hello world', 'test text'],
inputs: [
{
type: 'text',
input: 'hello world'
},
{
type: 'text',
input: 'test text'
}
],
type: 'db'
});
expect(mockVectorInsert).toHaveBeenCalledWith({
......@@ -282,7 +290,7 @@ describe('VectorDB Controller', () => {
});
it('should invalidate team vector cache after insert', async () => {
mockGetVectorsByText.mockResolvedValue({
mockGetVectors.mockResolvedValue({
tokens: 50,
vectors: [[0.1]]
});
......@@ -302,7 +310,7 @@ describe('VectorDB Controller', () => {
});
it('should return insert result when team vector cache invalidation fails', async () => {
mockGetVectorsByText.mockResolvedValue({
mockGetVectors.mockResolvedValue({
tokens: 50,
vectors: [[0.1]]
});
......@@ -332,7 +340,7 @@ describe('VectorDB Controller', () => {
it('should return insert result when team vector cache invalidation times out', async () => {
vi.useFakeTimers();
mockGetVectorsByText.mockResolvedValue({
mockGetVectors.mockResolvedValue({
tokens: 50,
vectors: [[0.1]]
});
......@@ -363,14 +371,6 @@ describe('VectorDB Controller', () => {
});
it('should handle empty inputs', async () => {
mockGetVectorsByText.mockResolvedValue({
tokens: 0,
vectors: []
});
mockVectorInsert.mockResolvedValue({
insertIds: []
});
const result = await insertDatasetDataVector({
teamId: 'team_123',
datasetId: 'dataset_456',
......@@ -383,6 +383,9 @@ describe('VectorDB Controller', () => {
tokens: 0,
insertIds: []
});
expect(mockGetVectors).not.toHaveBeenCalled();
expect(mockVectorInsert).not.toHaveBeenCalled();
expect(mockDelRedisCache).not.toHaveBeenCalled();
});
});
......
packages/service/test/core/ai/functions/queryExtension.test.ts
......@@ -73,6 +73,55 @@ describe('queryExtension', () => {
});
});
it('filters blank generated queries before cosine selection', async () => {
createLLMResponseMock.mockResolvedValue({
answerText: '[" ", " expanded query ", ""]',
requestId: 'req_query_extension_filter',
usage: {
inputTokens: 11,
outputTokens: 3,
usedUserOpenAIKey: false
}
});
await queryExtension({
query: 'original query',
histories: [],
llmModel: 'gpt-query',
embeddingModel: 'embedding-query'
});
expect(lazyGreedyQuerySelectionMock).toHaveBeenCalledWith({
originalText: 'original query',
candidates: ['expanded query'],
k: 1,
alpha: 0.3
});
});
it('does not run cosine selection when generated queries are all blank', async () => {
createLLMResponseMock.mockResolvedValue({
answerText: '[" ", ""]',
requestId: 'req_query_extension_blank',
usage: {
inputTokens: 11,
outputTokens: 3,
usedUserOpenAIKey: false
}
});
const result = await queryExtension({
query: 'original query',
histories: [],
llmModel: 'gpt-query',
embeddingModel: 'embedding-query'
});
expect(lazyGreedyQuerySelectionMock).not.toHaveBeenCalled();
expect(result.extensionQueries).toEqual([]);
expect(result.embeddingTokens).toBe(0);
});
it('splits fixed system prompt and dynamic user prompt', async () => {
filterGPTMessageByMaxContextMock.mockResolvedValue([
{
......
packages/service/test/core/ai/hooks/useTextCosine.test.ts
......@@ -5,7 +5,7 @@ import {
createMockVectorsResponse,
generateSimilarVector,
generateOrthogonalVector,
mockGetVectorsByText
mockGetVectors
} from '@test/mocks/core/ai/embedding';
describe('useTextCosine', () => {
......@@ -25,6 +25,8 @@ describe('useTextCosine', () => {
});
expect(result.selectedData).toEqual([]);
expect(result.embeddingTokens).toBe(0);
expect(mockGetVectors).not.toHaveBeenCalled();
});
it('should select k candidates when k <= candidates.length', async () => {
......@@ -73,7 +75,7 @@ describe('useTextCosine', () => {
// Create a candidate very different from original
const differentVector = generateOrthogonalVector(originalVector);
mockGetVectorsByText.mockResolvedValueOnce({
mockGetVectors.mockResolvedValueOnce({
tokens: 30,
vectors: [originalVector, differentVector, similarVector]
});
......@@ -110,7 +112,7 @@ describe('useTextCosine', () => {
expect(result.selectedData).toContain('c3');
});
it('should call getVectorsByText with correct parameters', async () => {
it('should call getVectors with correct parameters', async () => {
const { lazyGreedyQuerySelection } = useTextCosine({ embeddingModel: 'custom-model' });
await lazyGreedyQuerySelection({
originalText: 'test query',
......@@ -118,18 +120,59 @@ describe('useTextCosine', () => {
k: 1
});
expect(mockGetVectorsByText).toHaveBeenCalledWith({
expect(mockGetVectors).toHaveBeenCalledWith({
model: expect.anything(),
input: ['test query', 'candidate'],
inputs: [
{
type: 'text',
input: 'test query'
},
{
type: 'text',
input: 'candidate'
}
],
type: 'query'
});
});
it('should trim original text and skip blank candidates before embedding', async () => {
const originalVector = generateMockEmbedding('test query');
const candidateVector = generateSimilarVector(originalVector, 0.9);
mockGetVectors.mockResolvedValueOnce({
tokens: 10,
vectors: [originalVector, candidateVector]
});
const { lazyGreedyQuerySelection } = useTextCosine({ embeddingModel: 'custom-model' });
const result = await lazyGreedyQuerySelection({
originalText: ' test query ',
candidates: [' ', ' candidate ', ''],
k: 3
});
expect(mockGetVectors).toHaveBeenCalledWith({
model: expect.anything(),
inputs: [
{
type: 'text',
input: 'test query'
},
{
type: 'text',
input: 'candidate'
}
],
type: 'query'
});
expect(result.selectedData).toEqual(['candidate']);
});
it('should handle identical candidates correctly', async () => {
const originalVector = generateMockEmbedding('original');
const identicalVector = generateMockEmbedding('same');
mockGetVectorsByText.mockResolvedValueOnce({
mockGetVectors.mockResolvedValueOnce({
tokens: 30,
vectors: [originalVector, identicalVector, identicalVector, identicalVector]
});
......@@ -152,7 +195,7 @@ describe('useTextCosine', () => {
const similarVector = generateSimilarVector(originalVector, 0.9);
const differentVector = generateOrthogonalVector(originalVector);
mockGetVectorsByText.mockResolvedValueOnce({
mockGetVectors.mockResolvedValueOnce({
tokens: 25,
vectors: [originalVector, similarVector, differentVector]
});
......@@ -176,7 +219,7 @@ describe('useTextCosine', () => {
const mockResponse = createMockVectorsResponse(['test', 'candidate']);
mockResponse.tokens = 12345; // Override tokens for specific test
mockGetVectorsByText.mockResolvedValueOnce(mockResponse);
mockGetVectors.mockResolvedValueOnce(mockResponse);
const { lazyGreedyQuerySelection } = useTextCosine({
embeddingModel: 'text-embedding-ada-002'
......@@ -210,7 +253,7 @@ describe('useTextCosine', () => {
const similar2 = generateSimilarVector(similar1, 0.95); // Very close to similar1
const different = generateOrthogonalVector(originalVector);
mockGetVectorsByText.mockResolvedValueOnce({
mockGetVectors.mockResolvedValueOnce({
tokens: 40,
vectors: [originalVector, similar1, similar2, different]
});
......
packages/service/test/core/dataset/search/defaultRecall.test.ts 0 → 100644
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constants';
import { serviceEnv } from '@fastgpt/service/env';
const mockGetVectors = vi.hoisted(() => vi.fn());
const mockGetEmbeddingModel = vi.hoisted(() => vi.fn());
const mockGetDefaultRerankModel = vi.hoisted(() => vi.fn());
const mockGetLLMModel = vi.hoisted(() => vi.fn());
const mockIsImageEmbeddingModel = vi.hoisted(() => vi.fn());
const mockRecallFromVectorStore = vi.hoisted(() => vi.fn());
const mockCreateLLMResponse = vi.hoisted(() => vi.fn());
const mockMongoDatasetCollectionFind = vi.hoisted(() => vi.fn());
const mockMongoDatasetDataFind = vi.hoisted(() => vi.fn());
const mockMongoDatasetDataTextAggregate = vi.hoisted(() => vi.fn());
const mockGetImageBase64 = vi.hoisted(() => vi.fn());
const originalMultipleDataToBase64 = serviceEnv.MULTIPLE_DATA_TO_BASE64;
vi.mock('@fastgpt/service/core/ai/embedding', () => ({
getVectors: mockGetVectors
}));
vi.mock('@fastgpt/service/core/ai/model', () => ({
getEmbeddingModel: mockGetEmbeddingModel,
getDefaultRerankModel: mockGetDefaultRerankModel,
getLLMModel: mockGetLLMModel,
isImageEmbeddingModel: mockIsImageEmbeddingModel
}));
vi.mock('@fastgpt/service/common/vectorDB/controller', () => ({
recallFromVectorStore: mockRecallFromVectorStore
}));
vi.mock('@fastgpt/service/core/ai/llm/request', () => ({
createLLMResponse: mockCreateLLMResponse
}));
vi.mock('@fastgpt/service/common/file/image/utils', () => ({
getImageBase64: mockGetImageBase64
}));
vi.mock('@fastgpt/service/core/dataset/collection/schema', () => ({
DatasetColCollectionName: 'dataset_collections',
MongoDatasetCollection: {
find: mockMongoDatasetCollectionFind
}
}));
vi.mock('@fastgpt/service/core/dataset/data/schema', () => ({
DatasetDataCollectionName: 'dataset_datas',
MongoDatasetData: {
find: mockMongoDatasetDataFind
}
}));
vi.mock('@fastgpt/service/core/dataset/data/dataTextSchema', () => ({
MongoDatasetDataText: {
aggregate: mockMongoDatasetDataTextAggregate
}
}));
import { searchDatasetData } from '../../../../core/dataset/search/defaultRecall';
afterEach(() => {
serviceEnv.MULTIPLE_DATA_TO_BASE64 = originalMultipleDataToBase64;
});
describe('default recall dataset search', () => {
beforeEach(() => {
vi.clearAllMocks();
serviceEnv.MULTIPLE_DATA_TO_BASE64 = originalMultipleDataToBase64;
mockGetEmbeddingModel.mockReturnValue({
model: 'mock-embedding-model',
name: 'Mock Embedding Model'
});
mockGetDefaultRerankModel.mockReturnValue(undefined);
mockGetLLMModel.mockReturnValue({
model: 'mock-vlm-model',
name: 'Mock VLM Model',
vision: true
});
mockIsImageEmbeddingModel.mockReturnValue(false);
mockGetVectors.mockResolvedValue({
tokens: 10,
vectors: [
[0.1, 0.2],
[0.3, 0.4]
]
});
mockRecallFromVectorStore.mockResolvedValue({
results: []
});
mockMongoDatasetCollectionFind.mockImplementation((query: Record<string, any>) => {
if (query?.forbid) return [];
return {
lean: vi.fn().mockResolvedValue([])
};
});
mockMongoDatasetDataFind.mockReturnValue({
lean: vi.fn().mockResolvedValue([])
});
mockMongoDatasetDataTextAggregate.mockResolvedValue([]);
});
it('should ignore failed image caption and continue dataset search', async () => {
const userKey = { key: 'user-key', baseUrl: 'https://api.example.com/v1' };
mockCreateLLMResponse.mockRejectedValueOnce(new Error('vlm failed')).mockResolvedValueOnce({
requestId: 'req_image_caption_2',
answerText: 'red handbag on a white table',
usage: {
inputTokens: 3,
outputTokens: 2,
usedUserOpenAIKey: true
}
});
const result = await searchDatasetData({
histories: [],
teamId: 'team-1',
model: 'mock-embedding-model',
vlmModel: 'mock-vlm-model',
datasetIds: ['dataset-1'],
reRankQuery: 'black high heels',
textQueries: ['black high heels'],
imageQueries: ['data:image/png;base64,broken-image', 'data:image/png;base64,current-image'],
userKey,
limit: 5000,
searchMode: DatasetSearchModeEnum.embedding,
embeddingWeight: 0.5,
usingReRank: false
});
expect(result.imageCaptionResult).toEqual({
model: 'mock-vlm-model',
inputTokens: 3,
outputTokens: 2,
requestIds: ['req_image_caption_2'],
seconds: expect.any(Number),
usedUserOpenAIKey: true,
queries: ['red handbag on a white table']
});
expect(mockGetVectors).toHaveBeenCalledWith(
expect.objectContaining({
inputs: [
{
type: 'text',
input: 'black high heels'
},
{
type: 'text',
input: 'red handbag on a white table'
}
]
})
);
expect(mockCreateLLMResponse.mock.calls[1][0].body.messages[0].content[0].image_url.url).toBe(
'data:image/png;base64,current-image'
);
expect(mockCreateLLMResponse.mock.calls[1][0].userKey).toBe(userKey);
expect(result.searchRes).toEqual([]);
});
it('should request text and image embeddings in one getVectors call', async () => {
mockGetLLMModel.mockReturnValue(undefined);
mockIsImageEmbeddingModel.mockReturnValue(true);
mockGetVectors.mockResolvedValueOnce({
tokens: 12,
vectors: [
[0.1, 0.2],
[0.3, 0.4]
]
});
await searchDatasetData({
histories: [],
teamId: 'team-1',
model: 'mock-embedding-model',
datasetIds: ['dataset-1'],
reRankQuery: 'black high heels',
textQueries: ['black high heels'],
imageQueries: ['data:image/png;base64,current-image'],
limit: 5000,
searchMode: DatasetSearchModeEnum.embedding,
embeddingWeight: 0.5,
usingReRank: false
});
expect(mockGetVectors).toHaveBeenCalledTimes(1);
expect(mockGetVectors).toHaveBeenCalledWith(
expect.objectContaining({
inputs: [
{
type: 'text',
input: 'black high heels'
},
{
type: 'image',
input: 'data:image/png;base64,current-image'
}
]
})
);
expect(mockGetImageBase64).not.toHaveBeenCalledWith(' ');
});
it('should skip blank embedding recall inputs while preserving valid task order', async () => {
mockGetLLMModel.mockReturnValue(undefined);
mockIsImageEmbeddingModel.mockReturnValue(true);
mockGetVectors.mockResolvedValueOnce({
tokens: 12,
vectors: [
[0.1, 0.2],
[0.3, 0.4]
]
});
await searchDatasetData({
histories: [],
teamId: 'team-1',
model: 'mock-embedding-model',
datasetIds: ['dataset-1'],
reRankQuery: 'black high heels',
textQueries: [' ', ' black high heels '],
imageQueries: [' ', 'data:image/png;base64,current-image'],
limit: 5000,
searchMode: DatasetSearchModeEnum.embedding,
embeddingWeight: 0.5,
usingReRank: false
});
expect(mockGetVectors).toHaveBeenCalledWith(
expect.objectContaining({
inputs: [
{
type: 'text',
input: 'black high heels'
},
{
type: 'image',
input: 'data:image/png;base64,current-image'
}
]
})
);
});
it('should ignore failed image embedding normalization and keep text recall', async () => {
mockGetLLMModel.mockReturnValue(undefined);
mockIsImageEmbeddingModel.mockReturnValue(true);
serviceEnv.MULTIPLE_DATA_TO_BASE64 = true;
mockGetImageBase64.mockRejectedValueOnce(new Error('expired image'));
mockGetVectors.mockResolvedValueOnce({
tokens: 12,
vectors: [
[0.1, 0.2],
[0.3, 0.4]
]
});
await searchDatasetData({
histories: [],
teamId: 'team-1',
model: 'mock-embedding-model',
datasetIds: ['dataset-1'],
reRankQuery: 'black high heels',
textQueries: ['black high heels'],
imageQueries: [
'https://file.fastgpt.io/temp/team-1/expired.png?token=mock',
'data:image/png;base64,current-image'
],
limit: 5000,
searchMode: DatasetSearchModeEnum.embedding,
embeddingWeight: 0.5,
usingReRank: false
});
expect(mockGetVectors).toHaveBeenCalledWith(
expect.objectContaining({
inputs: [
{
type: 'text',
input: 'black high heels'
},
{
type: 'image',
input: 'data:image/png;base64,current-image'
}
]
})
);
});
it('should skip blank full-text queries before Mongo text search', async () => {
mockGetLLMModel.mockReturnValue(undefined);
mockIsImageEmbeddingModel.mockReturnValue(false);
const result = await searchDatasetData({
histories: [],
teamId: 'team-1',
model: 'mock-embedding-model',
datasetIds: ['dataset-1'],
reRankQuery: '',
textQueries: [' ', '\n'],
imageQueries: [],
limit: 5000,
searchMode: DatasetSearchModeEnum.fullTextRecall,
embeddingWeight: 0.5,
usingReRank: false
});
expect(mockGetVectors).not.toHaveBeenCalled();
expect(mockMongoDatasetDataTextAggregate).not.toHaveBeenCalled();
expect(result.searchRes).toEqual([]);
});
});
packages/service/test/core/dataset/search/index.test.ts 0 → 100644
import { beforeEach, describe, expect, it, vi } from 'vitest';
const mockSearchDatasetData = vi.hoisted(() => vi.fn());
const mockDatasetSearchQueryExtension = vi.hoisted(() => vi.fn());
const mockGetLLMModel = vi.hoisted(() => vi.fn());
vi.mock('@fastgpt/service/core/dataset/search/defaultRecall', () => ({
searchDatasetData: mockSearchDatasetData
}));
vi.mock('@fastgpt/service/core/dataset/search/utils', () => ({
datasetSearchQueryExtension: mockDatasetSearchQueryExtension
}));
vi.mock('@fastgpt/service/core/ai/model', () => ({
getLLMModel: mockGetLLMModel
}));
import { defaultSearchDatasetData } from '../../../../core/dataset/search';
describe('defaultSearchDatasetData', () => {
beforeEach(() => {
vi.clearAllMocks();
mockSearchDatasetData.mockResolvedValue({
searchRes: [],
embeddingTokens: 0,
reRankInputTokens: 0,
searchMode: 'embedding',
limit: 5000,
similarity: 0,
usingReRank: false,
usingSimilarityFilter: false
});
});
it('should drop whitespace-only text queries before extension and recall', async () => {
await defaultSearchDatasetData({
histories: [],
teamId: 'team-1',
datasetIds: ['dataset-1'],
model: 'embedding-model',
textQueries: [' ', '\n'],
imageQueries: ['https://files.example.com/query.png'],
limit: 5000,
datasetSearchUsingExtensionQuery: true
});
expect(mockDatasetSearchQueryExtension).not.toHaveBeenCalled();
expect(mockSearchDatasetData).toHaveBeenCalledWith(
expect.objectContaining({
reRankQuery: '',
textQueries: []
})
);
});
it('should trim text queries before query extension', async () => {
mockGetLLMModel.mockReturnValue({
model: 'query-extension-model'
});
mockDatasetSearchQueryExtension.mockResolvedValue({
searchQueries: ['first', 'second', 'expanded'],
reRankQuery: 'first\nsecond\nexpanded',
aiExtensionResult: undefined
});
await defaultSearchDatasetData({
histories: [],
teamId: 'team-1',
datasetIds: ['dataset-1'],
model: 'embedding-model',
textQueries: [' first ', ' ', 'second'],
imageQueries: [],
limit: 5000,
datasetSearchUsingExtensionQuery: true,
datasetSearchExtensionModel: 'query-extension-model'
});
expect(mockDatasetSearchQueryExtension).toHaveBeenCalledWith(
expect.objectContaining({
query: 'first\nsecond',
llmModel: 'query-extension-model'
})
);
expect(mockSearchDatasetData).toHaveBeenCalledWith(
expect.objectContaining({
reRankQuery: 'first\nsecond\nexpanded',
textQueries: ['first', 'second', 'expanded']
})
);
});
});
packages/service/test/core/workflow/dispatch/ai/agent/sub/dataset.test.ts
......@@ -16,7 +16,7 @@ const {
formatModelChars2PointsMock: vi.fn()
}));
vi.mock('@fastgpt/service/core/dataset/search/controller', () => ({
vi.mock('@fastgpt/service/core/dataset/search', () => ({
defaultSearchDatasetData: defaultSearchDatasetDataMock
}));
......
packages/service/test/core/workflow/dispatch/ai/toolcall/hooks/useToolRunner.test.ts
import { beforeEach, describe, expect, it, vi } from 'vitest';
import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
import { useToolRunner } from '@fastgpt/service/core/workflow/dispatch/ai/toolcall/hooks/useToolRunner';
......@@ -78,12 +79,14 @@ const createRunner = ({
getToolInfo,
runtimeNodes = [],
runtimeEdges = [],
allFiles = new Map()
allFiles = new Map(),
fileUrls = []
}: {
getToolInfo: (name: string) => any;
runtimeNodes?: any[];
runtimeEdges?: any[];
allFiles?: Map<string, any>;
fileUrls?: string[];
}) => {
const cacheToolFlowResponse = vi.fn();
const appendToolFlowResponse = vi.fn();
......@@ -93,6 +96,7 @@ const createRunner = ({
runtimeNodes,
runtimeEdges,
allFiles,
fileUrls,
getToolInfo,
cacheToolFlowResponse,
appendToolFlowResponse,
......@@ -254,6 +258,92 @@ describe('useToolRunner', () => {
});
});
it('injects parent file urls into dataset search tool calls', async () => {
const runtimeNodes = [
{
nodeId: 'dataset_search',
inputs: [
{
key: NodeInputKeyEnum.userChatInput,
value: 'legacy default'
},
{
key: NodeInputKeyEnum.datasetSearchInput,
value: []
},
{
key: 'limit',
value: 10
}
]
}
];
const runtimeEdges = [
{
target: 'dataset_search'
}
];
runWorkflowMock.mockResolvedValue({
toolResponses: 'dataset ok',
assistantResponses: [],
flowUsages: [],
workflowInteractiveResponse: undefined,
flowResponses: []
});
const { runTool } = createRunner({
runtimeNodes,
runtimeEdges,
fileUrls: ['https://files/image.png'],
getToolInfo: () => ({
type: 'user',
name: 'Dataset search',
avatar: 'dataset-avatar',
rawData: {
nodeId: 'dataset_search',
flowNodeType: FlowNodeTypeEnum.datasetSearchNode
}
})
});
const call = createCall({
id: 'call_dataset_search',
name: 'dataset_search',
args: '{"datasetSearchInput":"red shoes","limit":3}'
});
const result = await runTool({ call });
expect(runtimeNodes[0]).toEqual({
nodeId: 'dataset_search',
isEntry: true,
inputs: [
{
key: NodeInputKeyEnum.userChatInput,
value: ''
},
{
key: NodeInputKeyEnum.datasetSearchInput,
value: ['red shoes', 'https://files/image.png']
},
{
key: 'limit',
value: 3
}
]
});
expect(runtimeEdges[0]).toEqual({
target: 'dataset_search',
status: 'active'
});
expect(result).toEqual({
response: 'dataset ok',
assistantMessages: [],
usages: [],
interactive: undefined,
stop: false
});
});
it('runs user workflow tools and interactive resume paths', async () => {
const usage = {
moduleName: 'tool',
......
packages/service/test/core/workflow/dispatch/dataset/search.test.ts
......@@ -5,19 +5,21 @@ import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
const {
defaultSearchDatasetDataMock,
deepRagSearchMock,
findDatasetByIdMock,
formatModelChars2PointsMock,
usagePushMock
} = vi.hoisted(() => ({
defaultSearchDatasetDataMock: vi.fn(),
deepRagSearchMock: vi.fn(),
findDatasetByIdMock: vi.fn(),
formatModelChars2PointsMock: vi.fn(),
usagePushMock: vi.fn()
}));
vi.mock('@fastgpt/service/core/dataset/search/controller', () => ({
vi.mock('@fastgpt/service/core/dataset/search', () => ({
defaultSearchDatasetData: defaultSearchDatasetDataMock,
deepRagSearch: vi.fn()
deepRagSearch: deepRagSearchMock
}));
vi.mock('@fastgpt/service/core/dataset/schema', () => ({
......@@ -208,4 +210,123 @@ describe('dispatchDatasetSearch', () => {
])
);
});
it('adds image caption request ids and skips platform points when external OpenAI key is used', async () => {
const userKey = { key: 'user-key', baseUrl: 'https://api.example.com/v1' };
defaultSearchDatasetDataMock.mockResolvedValue({
searchRes: [],
embeddingTokens: 20,
reRankInputTokens: 0,
usingSimilarityFilter: true,
usingReRank: false,
imageCaptionResult: {
model: 'gpt-vision',
inputTokens: 4,
outputTokens: 3,
requestIds: ['req_image_caption_1', 'req_image_caption_2'],
seconds: 1.5,
usedUserOpenAIKey: true,
queries: ['red handbag', 'blue sneaker']
}
});
const result = await dispatchDatasetSearch({
runningAppInfo: { teamId: 'team_1' },
runningUserInfo: { tmbId: 'tmb_1' },
externalProvider: {
openaiAccount: userKey
},
histories: [],
node: { name: 'Dataset Search' },
params: {
datasets: [{ datasetId: 'dataset_1' }],
similarity: 0.4,
limit: 5000,
userChatInput: '',
datasetSearchInput: ['https://files.example.com/query.png'],
authTmbId: false,
searchMode: DatasetSearchModeEnum.embedding,
embeddingWeight: 0.5,
usingReRank: false,
rerankWeight: 0.5,
datasetSearchUsingExtensionQuery: false
},
usagePush: usagePushMock
} as any);
expect(defaultSearchDatasetDataMock).toHaveBeenCalledWith(
expect.objectContaining({
userKey,
imageQueries: ['https://files.example.com/query.png']
})
);
const nodeResponse = result[DispatchNodeResponseKeyEnum.nodeResponse];
expect(nodeResponse?.childrenResponses?.[0]).toMatchObject({
id: 'req_image_caption_1',
nodeId: 'req_image_caption_1',
moduleType: FlowNodeTypeEnum.datasetSearchNode,
moduleName: 'account_usage:image_parse',
moduleLogo: 'core/workflow/template/datasetSearch',
runningTime: 1.5,
model: 'gpt-vision name',
llmRequestIds: ['req_image_caption_1', 'req_image_caption_2'],
inputTokens: 4,
outputTokens: 3,
totalPoints: 0,
textOutput: 'red handbag\nblue sneaker'
});
expect(usagePushMock).toHaveBeenCalledWith(
expect.arrayContaining([
expect.objectContaining({
moduleName: 'account_usage:image_parse',
totalPoints: 0
})
])
);
});
it('uses default recall for image-only input even when deep search is enabled', async () => {
defaultSearchDatasetDataMock.mockResolvedValue({
searchRes: [],
embeddingTokens: 4,
reRankInputTokens: 0,
usingSimilarityFilter: false,
usingReRank: false
});
const result = await dispatchDatasetSearch({
runningAppInfo: { teamId: 'team_1' },
runningUserInfo: { tmbId: 'tmb_1' },
externalProvider: {},
histories: [],
node: { name: 'Dataset Search' },
params: {
datasets: [{ datasetId: 'dataset_1' }],
similarity: 0.4,
limit: 5000,
userChatInput: '',
datasetSearchInput: ['https://files.example.com/query.png'],
authTmbId: false,
searchMode: DatasetSearchModeEnum.embedding,
embeddingWeight: 0.5,
usingReRank: false,
rerankWeight: 0.5,
datasetSearchUsingExtensionQuery: false,
datasetDeepSearch: true
},
usagePush: usagePushMock
} as any);
expect(deepRagSearchMock).not.toHaveBeenCalled();
expect(defaultSearchDatasetDataMock).toHaveBeenCalledWith(
expect.objectContaining({
textQueries: [],
imageQueries: ['https://files.example.com/query.png']
})
);
expect(result[DispatchNodeResponseKeyEnum.nodeResponse]?.datasetQueries).toEqual([
'https://files.example.com/query.png'
]);
});
});
packages/service/test/core/workflow/dispatch/dataset/utils.test.ts 0 → 100644
import { describe, expect, it } from 'vitest';
import { normalizeDatasetSearchInput } from '../../../../../core/workflow/dispatch/dataset/utils';
describe('normalizeDatasetSearchInput', () => {
it('should split text queries and http image urls', () => {
const result = normalizeDatasetSearchInput([
' black high heels ',
'temp/team-1/search-image.png',
'chat/team-1/manual.pdf',
'https://example.com/current.png',
'data:image/png;base64,abc',
'dataset/team-1/photo.webp',
'https://example.com/manual.pdf',
''
]);
expect(result.textQueries).toEqual([
'black high heels',
'temp/team-1/search-image.png',
'chat/team-1/manual.pdf',
'data:image/png;base64,abc',
'dataset/team-1/photo.webp'
]);
expect(result.imageQueries).toEqual(['https://example.com/current.png']);
});
it('should only classify http image urls by parsed file type', () => {
const result = normalizeDatasetSearchInput([
'/api/file/read?filename=current.png',
'/api/file/read?filename=manual.pdf',
'https://cdn.example.com/download?filename=current.png',
'https://cdn.example.com/download?filename=current.png'
]);
expect(result.textQueries).toEqual([
'/api/file/read?filename=current.png',
'/api/file/read?filename=manual.pdf'
]);
expect(result.imageQueries).toEqual(['https://cdn.example.com/download?filename=current.png']);
});
it('should remove duplicated text queries and images', () => {
const result = normalizeDatasetSearchInput([
'same query',
'same query',
'https://example.com/a.png',
'https://example.com/a.png'
]);
expect(result.textQueries).toEqual(['same query']);
expect(result.imageQueries).toEqual(['https://example.com/a.png']);
});
});
packages/service/test/core/workflow/dispatch/index.test.ts
......@@ -559,7 +559,7 @@ describe('WorkflowQueue', () => {
const group1 = [createEdge('A', 'D', 'waiting')];
const group2 = [createEdge('B', 'D', 'skipped'), createEdge('C', 'D', 'skipped')];
const nodeEdgeGroupsMap = new Map([['D', [group1, group2]]]);
console.log(nodeEdgeGroupsMap);
const result = WorkflowQueue.getNodeRunStatus({ node, nodeEdgeGroupsMap });
expect(result).toBe('wait');
......
packages/service/test/core/workflow/utils/context.test.ts
......@@ -3,15 +3,21 @@ import {
parseUrlToFileType,
runWithContext,
getWorkflowContext,
updateWorkflowContextVal,
WorkflowContext
} from '@fastgpt/service/core/workflow/utils/context';
updateWorkflowContextVal
} from '../../../../core/workflow/utils/context';
import { ChatFileTypeEnum } from '@fastgpt/global/core/chat/constants';
const createWorkflowContext = (queryUrlTypeMap: Record<string, ChatFileTypeEnum>) => ({
queryUrlTypeMap,
mcpClientMemory: {}
});
describe('WorkflowContext', () => {
describe('runWithContext / getWorkflowContext', () => {
it('should provide context inside callback', () => {
const ctx = { queryUrlTypeMap: { 'http://a.com/f.pdf': ChatFileTypeEnum.file } };
const ctx = createWorkflowContext({
'http://a.com/f.pdf': ChatFileTypeEnum.file
});
runWithContext(ctx, () => {
const store = getWorkflowContext();
......@@ -25,8 +31,8 @@ describe('WorkflowContext', () => {
});
it('should isolate nested contexts', () => {
const outer = { queryUrlTypeMap: { a: ChatFileTypeEnum.file } };
const inner = { queryUrlTypeMap: { b: ChatFileTypeEnum.image } };
const outer = createWorkflowContext({ a: ChatFileTypeEnum.file });
const inner = createWorkflowContext({ b: ChatFileTypeEnum.image });
runWithContext(outer, () => {
expect(getWorkflowContext()?.queryUrlTypeMap).toEqual(outer.queryUrlTypeMap);
......@@ -41,7 +47,7 @@ describe('WorkflowContext', () => {
});
it('should work with async functions', async () => {
const ctx = { queryUrlTypeMap: { url1: ChatFileTypeEnum.image } };
const ctx = createWorkflowContext({ url1: ChatFileTypeEnum.image });
await new Promise<void>((resolve) => {
runWithContext(ctx, async () => {
......@@ -55,7 +61,7 @@ describe('WorkflowContext', () => {
describe('updateWorkflowContextVal', () => {
it('should update existing context values', () => {
const ctx = { queryUrlTypeMap: { a: ChatFileTypeEnum.file } };
const ctx = createWorkflowContext({ a: ChatFileTypeEnum.file });
runWithContext(ctx, () => {
updateWorkflowContextVal({
......@@ -75,7 +81,7 @@ describe('WorkflowContext', () => {
});
it('should support partial updates', () => {
const ctx = { queryUrlTypeMap: { a: ChatFileTypeEnum.file } };
const ctx = createWorkflowContext({ a: ChatFileTypeEnum.file });
runWithContext(ctx, () => {
// Update with empty partial — no keys iterated
......@@ -88,7 +94,7 @@ describe('WorkflowContext', () => {
describe('parseUrlToFileType with context', () => {
it('should use queryUrlTypeMap to determine file type', () => {
const url = 'https://example.com/unknown-resource';
const ctx = { queryUrlTypeMap: { [url]: ChatFileTypeEnum.image } };
const ctx = createWorkflowContext({ [url]: ChatFileTypeEnum.image });
runWithContext(ctx, () => {
const result = parseUrlToFileType(url);
......@@ -98,7 +104,7 @@ describe('WorkflowContext', () => {
it('should prefer context type over extension-based detection', () => {
const url = 'https://example.com/photo.png';
const ctx = { queryUrlTypeMap: { [url]: ChatFileTypeEnum.file } };
const ctx = createWorkflowContext({ [url]: ChatFileTypeEnum.file });
runWithContext(ctx, () => {
const result = parseUrlToFileType(url);
......@@ -108,7 +114,7 @@ describe('WorkflowContext', () => {
});
it('should fall back to extension detection when URL not in context', () => {
const ctx = { queryUrlTypeMap: { 'other-url': ChatFileTypeEnum.file } };
const ctx = createWorkflowContext({ 'other-url': ChatFileTypeEnum.file });
runWithContext(ctx, () => {
const result = parseUrlToFileType('https://example.com/photo.png');
......@@ -339,21 +345,21 @@ describe('parseUrlToFileType', () => {
describe('edge cases', () => {
it('should return undefined for non-string input', () => {
// @ts-ignore - testing runtime behavior
// @ts-expect-error testing runtime behavior
const result = parseUrlToFileType(123);
expect(result).toBeUndefined();
});
it('should return undefined for null', () => {
// @ts-ignore - testing runtime behavior
// @ts-expect-error testing runtime behavior
const result = parseUrlToFileType(null);
expect(result).toBeUndefined();
});
it('should return undefined for undefined', () => {
// @ts-ignore - testing runtime behavior
// @ts-expect-error testing runtime behavior
const result = parseUrlToFileType(undefined);
expect(result).toBeUndefined();
......
packages/service/vitest.config.ts
......@@ -23,7 +23,7 @@ export default defineConfig({
},
coverage: {
enabled: true,
reporter: ['text', 'text-summary', 'html', 'json-summary', 'json'],
reporter: ['html', 'json-summary', 'json'],
reportOnFailure: true,
include: ['common/**/*.ts', 'core/**/*.ts', 'support/**/*.ts', 'worker/**/*.ts'],
exclude: [
......
packages/web/components/common/Image/PhotoView.tsx
import React from 'react';
import { PhotoProvider, PhotoView } from 'react-photo-view';
import { PhotoProvider, PhotoSlider, PhotoView } from 'react-photo-view';
import 'react-photo-view/dist/react-photo-view.css';
import { type ImageProps } from '@chakra-ui/react';
import { useSystem } from '../../../hooks/useSystem';
import Loading from '../MyLoading';
import MyImage from './MyImage';
type MyPhotoSliderProps = {
src?: string;
visible: boolean;
onClose: () => void;
imageKey?: string;
};
const MyPhotoView = (props: ImageProps) => {
const { isPc } = useSystem();
......@@ -23,4 +30,20 @@ const MyPhotoView = (props: ImageProps) => {
);
};
export const MyPhotoSlider = ({ src, visible, onClose, imageKey }: MyPhotoSliderProps) => {
const { isPc } = useSystem();
return (
<PhotoSlider
images={src ? [{ key: imageKey || src, src }] : []}
visible={visible}
onClose={onClose}
maskOpacity={0.6}
bannerVisible={!isPc}
photoClosable
loadingElement={<Loading fixed={false} />}
/>
);
};
export default MyPhotoView;
pro @ fa441cde
Subproject commit cde23cd5f0cda7d89aa8b9ea66b51fa2f0d4fa2d
Subproject commit fa441cde179aac9113f67941216c8d990c5ba681
projects/app/src/components/core/chat/components/WholeResponseModal/ImageQuery.tsx
import ImagePreviewToken from '@/components/core/dataset/ImagePreviewToken';
import type { ImagePreviewTokenItemType } from '@/components/core/dataset/ImagePreviewToken';
import { Box } from '@chakra-ui/react';
import type { ChatHistoryItemResType } from '@fastgpt/global/core/chat/type';
import React from 'react';
const httpUrlReg = /^https?:\/\//i;
/**
* nodeResponse 里为了兼容历史展示，把文本 query 和图片 URL 都放在 datasetQueries。
* UI 侧只负责展示拆分：普通 http(s) URL 作为图片 token，其余内容按原文本换行展示。
*/
const splitDatasetQueries = (datasetQueries: string[]) => {
const textQueries: string[] = [];
const queryImages: ImagePreviewTokenItemType[] = [];
for (const query of datasetQueries) {
if (!query) continue;
if (httpUrlReg.test(query)) {
queryImages.push({ url: query });
} else {
textQueries.push(query);
}
}
return {
query: textQueries.join('\n'),
queryImages
};
};
const ImageQuery = ({
query,
queryImages,
datasetQueries,
datasetId
}: {
query?: string;
queryImages: NonNullable<ChatHistoryItemResType['queryImages']>;
datasetQueries: string[];
datasetId?: string;
}) => {
const { query, queryImages } = splitDatasetQueries(datasetQueries);
return (
<Box
border={'1px solid'}
......
projects/app/src/components/core/chat/components/WholeResponseModal/ResponseRows.tsx
......@@ -167,20 +167,7 @@ export const AiChatRows = ({
/>
<Row label={t('chat:step_query')} value={activeModule.stepQuery} />
{activeModule.queryImages && activeModule.queryImages.length > 0 ? (
<Row
label={t('common:core.chat.response.module query')}
rawDom={
<ImageQuery
query={activeModule.query}
queryImages={activeModule.queryImages}
datasetId={queryPreviewDatasetId}
/>
}
/>
) : (
<Row label={t('common:core.chat.response.module query')} value={activeModule.query} />
)}
<Row label={t('common:core.chat.response.module query')} value={activeModule.query} />
<Row
label={t('common:core.chat.response.context total length')}
value={activeModule.contextTotalLen}
......@@ -251,6 +238,17 @@ export const DatasetSearchRows = ({
label={t('common:core.chat.response.module similarity')}
value={activeModule.similarity}
/>
{activeModule.datasetQueries && activeModule.datasetQueries.length > 0 && (
<Row
label={t('common:core.chat.response.module query')}
rawDom={
<ImageQuery
datasetQueries={activeModule.datasetQueries}
datasetId={activeModule.quoteList?.[0]?.datasetId}
/>
}
/>
)}
<Row label={t('common:core.chat.response.module limit')} value={activeModule.limit} />
<Row label={t('chat:response_embedding_model')} value={activeModule.embeddingModel} />
<Row
......
projects/app/src/pageComponents/app/detail/Edit/SimpleApp/utils.ts
......@@ -385,7 +385,7 @@ export function form2AppWorkflow(
return {
nodeId: datasetNodeId,
name: t(DatasetSearchModule.name),
intro: t('app:dataset_search_tool_description'),
intro: DatasetSearchModule.intro,
avatar: DatasetSearchModule.avatar,
flowNodeType: DatasetSearchModule.flowNodeType,
showStatus: true,
......@@ -476,6 +476,8 @@ export function form2AppWorkflow(
},
{
...Input_Template_UserChatInput,
key: NodeInputKeyEnum.datasetSearchInput,
valueType: WorkflowIOValueTypeEnum.arrayString,
toolDescription: i18nT('workflow:content_to_search'),
value: question
}
......@@ -503,7 +505,10 @@ export function form2AppWorkflow(
return {
nodes: [
aiChatTemplate(formData),
datasetNodeTemplate(formData, [workflowStartNodeId, 'userChatInput'])
datasetNodeTemplate(formData, [
[workflowStartNodeId, NodeOutputKeyEnum.userChatInput],
[workflowStartNodeId, NodeOutputKeyEnum.userFiles]
])
],
edges: [
{
......
projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/components/NodeTemplates/list.tsx
......@@ -264,6 +264,7 @@ const NodeTemplateList = ({
const defaultValueMap: Record<string, any> = {
[NodeInputKeyEnum.userChatInput]: undefined,
[NodeInputKeyEnum.datasetSearchInput]: undefined,
[NodeInputKeyEnum.fileUrlList]: undefined
};
......@@ -276,6 +277,10 @@ const NodeTemplateList = ({
defaultValueMap[NodeInputKeyEnum.fileUrlList] = [
[node.nodeId, NodeOutputKeyEnum.userFiles]
];
defaultValueMap[NodeInputKeyEnum.datasetSearchInput] = [
[node.nodeId, NodeOutputKeyEnum.userChatInput],
[node.nodeId, NodeOutputKeyEnum.userFiles]
];
}
});
......@@ -413,7 +418,7 @@ const NodeTemplateList = ({
[
computedNewNodeName,
getNodeById,
handleParams?.nodeId,
handleParams,
getNodeList,
getIntersectingNodes,
onAddNode,
......@@ -488,7 +493,7 @@ const NodeTemplateList = ({
return data.filter(({ list }) => list.length > 0);
}, [templateType, templates, t, i18n.language]);
const NodeListRender = useMemoizedFn(({ list = [] }: { list: NodeTemplateListType }) => {
const renderNodeList = useMemoizedFn((list: NodeTemplateListType = []) => {
return (
<>
{list.map((item) => {
......@@ -559,14 +564,12 @@ const NodeTemplateList = ({
{t(label as any)}
<AccordionIcon />
</AccordionButton>
<AccordionPanel py={0}>
<NodeListRender list={list} />
</AccordionPanel>
<AccordionPanel py={0}>{renderNodeList(list)}</AccordionPanel>
</AccordionItem>
))}
</>
) : (
<NodeListRender list={formatTemplatesArrayData?.[0]?.list} />
<>{renderNodeList(formatTemplatesArrayData?.[0]?.list)}</>
)}
</Accordion>
</Box>
......
projects/app/src/pageComponents/dataset/detail/Test/components/TestHistories.tsx
......@@ -52,10 +52,10 @@ const TestHistories = ({
borderColor: 'primary.300',
boxShadow: '1',
'& .delete': {
display: 'block'
visibility: 'visible'
},
'& .time': {
display: 'none'
visibility: 'hidden'
}
}}
cursor={'pointer'}
......@@ -97,7 +97,7 @@ const TestHistories = ({
{t(formatTimeToChatTime(item.time) as any).replace('#', ':')}
</Box>
<MyTooltip label={t('common:core.dataset.test.delete test history')}>
<Box className="delete" display={'none'} w={'0.8rem'} h={'0.8rem'} ml={1}>
<Box className="delete" visibility={'hidden'} w={'0.8rem'} h={'0.8rem'} ml={1}>
<MyIcon
name={'delete'}
w={'0.8rem'}
......
projects/app/src/pageComponents/dataset/detail/Test/constants.ts
import { imageFileType } from '@fastgpt/global/common/file/constants';
// Keep the disabled upload path in place so the search test flow can enable images by flag only.
export const SEARCH_TEST_IMAGE_UPLOAD_ENABLED = false;
export const SEARCH_TEST_IMAGE_UPLOAD_ENABLED = true;
export const MAX_SEARCH_TEST_IMAGE_COUNT = 10;
// Derive accepted extensions from the shared upload accept string to avoid frontend/backend drift.
......
projects/app/src/pageComponents/dataset/detail/Test/hooks/useDatasetSearchTest.ts
......@@ -79,9 +79,7 @@ export const useDatasetSearchTest = ({
text: getValues('inputText').trim(),
time: new Date(),
results: res.list,
// Histories only need stable image keys; preview urls are regenerated by ImagePreviewToken.
queryImageRefs:
queryImageRefs.length > 0 ? queryImageRefs.map(({ key }) => ({ key })) : undefined,
queryImageRefs: queryImageRefs.length > 0 ? queryImageRefs : undefined,
duration: res.duration,
searchMode: res.searchMode,
usingReRank: res.usingReRank,
......
projects/app/src/pages/api/core/ai/model/test.ts
......@@ -11,7 +11,7 @@ import {
} from '@fastgpt/global/core/ai/model.schema';
import { getAIApi } from '@fastgpt/service/core/ai/config';
import { getLogger, LogCategories } from '@fastgpt/service/common/logger';
import { getVectorsByText } from '@fastgpt/service/core/ai/embedding';
import { getVectors } from '@fastgpt/service/core/ai/embedding';
import { reRankRecall } from '@fastgpt/service/core/ai/rerank';
import { aiTranscriptions } from '@fastgpt/service/core/ai/audio/transcriptions';
import { isProduction } from '@fastgpt/global/common/system/constants';
......@@ -87,9 +87,14 @@ const testEmbeddingModel = async (
model: EmbeddingModelItemType,
headers: Record<string, string>
) => {
return getVectorsByText({
input: 'Hi',
return getVectors({
model,
inputs: [
{
type: 'text',
input: 'Hi'
}
],
headers
});
};
......
projects/app/src/pages/api/core/dataset/searchTest.ts
import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
import { pushDatasetTestUsage } from '@/service/support/wallet/usage/push';
import {
deepRagSearch,
defaultSearchDatasetData
} from '@fastgpt/service/core/dataset/search/controller';
import { deepRagSearch, defaultSearchDatasetData } from '@fastgpt/service/core/dataset/search';
import { updateApiKeyUsage } from '@fastgpt/service/support/openapi/tools';
import { UsageSourceEnum } from '@fastgpt/global/support/wallet/usage/constants';
import { checkTeamAIPoints } from '@fastgpt/service/support/permission/teamLimit';
......@@ -15,6 +12,8 @@ import { getRerankModel } from '@fastgpt/service/core/ai/model';
import { addAuditLog } from '@fastgpt/service/support/user/audit/util';
import { AuditEventEnum } from '@fastgpt/global/support/user/audit/constants';
import { getI18nDatasetType } from '@fastgpt/service/support/user/audit/util';
import { isS3ObjectKey } from '@fastgpt/service/common/s3/utils';
import { getS3DatasetSource } from '@fastgpt/service/common/s3/sources/dataset';
import {
SearchDatasetTestBodySchema,
SearchDatasetTestResponseSchema,
......@@ -22,12 +21,13 @@ import {
type SearchDatasetTestResponse
} from '@fastgpt/global/openapi/core/dataset/api';
async function handler(
export async function handler(
req: ApiRequestProps<SearchDatasetTestBody>
): Promise<SearchDatasetTestResponse> {
const {
datasetId,
text,
queryImageUrls,
limit = 5000,
similarity,
searchMode,
......@@ -50,7 +50,7 @@ async function handler(
const start = Date.now();
// auth dataset role
const { dataset, teamId, tmbId, userId, apikey } = await authDataset({
const { dataset, teamId, tmbId, apikey } = await authDataset({
req,
authToken: true,
authApiKey: true,
......@@ -60,14 +60,37 @@ async function handler(
// auth balance
await checkTeamAIPoints(teamId);
// Search-test images must be temp objects created by this team. Client-supplied keys are not
// proof of ownership, so reject dataset/chat/foreign-team keys before any S3 read happens.
const validQueryImageKeys = queryImageUrls.filter(
(key) => isS3ObjectKey(key, 'temp') && key.startsWith(`temp/${teamId}/`)
);
if (validQueryImageKeys.length !== queryImageUrls.length) {
return Promise.reject('Invalid query image key');
}
// 搜索主链路只接收模型可读图片 URL；temp key 的鉴权和临时 URL 生成固定在入口层完成。
const validQueryImageUrls = await Promise.all(
validQueryImageKeys.map(async (key) => {
const { url } = await getS3DatasetSource().createExternalUrl({
key,
expiredHours: 1
});
return url;
})
);
const rerankModelData = getRerankModel(rerankModel);
const searchData = {
histories: [],
teamId,
reRankQuery: text,
queries: [text],
textQueries: text ? [text] : [],
imageQueries: validQueryImageUrls,
model: dataset.vectorModel,
vlmModel: dataset.vlmModel,
limit: Math.min(limit, 20000),
similarity,
datasetIds: [datasetId],
......@@ -83,9 +106,9 @@ async function handler(
reRankInputTokens,
usingReRank: searchUsingReRank,
queryExtensionResult,
deepSearchResult,
imageCaptionResult,
...result
} = datasetDeepSearch
} = datasetDeepSearch && !!text.trim()
? await deepRagSearch({
...searchData,
datasetDeepSearchModel,
......@@ -123,6 +146,13 @@ async function handler(
embeddingTokens: queryExtensionResult.embeddingTokens,
embeddingModel: dataset.vectorModel
}
: undefined,
imageCaptionUsage: imageCaptionResult
? {
model: imageCaptionResult.model,
inputTokens: imageCaptionResult.inputTokens,
outputTokens: imageCaptionResult.outputTokens
}
: undefined
});
......
projects/app/src/service/common/system/index.ts
......@@ -12,7 +12,7 @@ import { POST } from '@fastgpt/service/common/api/plusRequest';
import {
type DeepRagSearchProps,
type SearchDatasetDataResponse
} from '@fastgpt/service/core/dataset/search/controller';
} from '@fastgpt/service/core/dataset/search';
import { type AuthOpenApiLimitProps } from '@fastgpt/service/support/openapi/auth';
import type {
PushUsageItemsProps,
......@@ -219,14 +219,15 @@ export async function initAppTemplateTypes() {
try {
await Promise.all(
defaultTemplateTypes.map((templateType) => {
const { typeOrder, ...rest } = templateType;
return MongoTemplateTypes.updateOne(
{
typeId: templateType.typeId
},
{
$set: rest
$set: {
typeId: templateType.typeId,
typeName: templateType.typeName
}
},
{
upsert: true
......
projects/app/src/service/support/wallet/usage/push.ts
......@@ -280,7 +280,8 @@ export const pushDatasetTestUsage = ({
source = UsageSourceEnum.fastgpt,
embUsage,
rerankUsage,
extensionUsage
extensionUsage,
imageCaptionUsage
}: {
teamId: string;
tmbId: string;
......@@ -300,6 +301,11 @@ export const pushDatasetTestUsage = ({
embeddingTokens: number;
embeddingModel: string;
};
imageCaptionUsage?: {
model: string;
inputTokens: number;
outputTokens: number;
};
}) => {
const list: UsageItemType[] = [];
let points = 0;
......@@ -359,6 +365,21 @@ export const pushDatasetTestUsage = ({
inputTokens: rerankUsage.inputTokens
});
}
if (imageCaptionUsage) {
const { totalPoints, modelName } = formatModelChars2Points({
model: imageCaptionUsage.model,
inputTokens: imageCaptionUsage.inputTokens,
outputTokens: imageCaptionUsage.outputTokens
});
points += totalPoints;
list.push({
moduleName: i18nT('account_usage:image_parse'),
amount: totalPoints,
model: modelName,
inputTokens: imageCaptionUsage.inputTokens,
outputTokens: imageCaptionUsage.outputTokens
});
}
createUsage({
teamId,
......
projects/app/src/web/core/workflow/adapt.ts
......@@ -475,7 +475,20 @@ export const v1Workflow2V2 = (
// 更新特殊的输入(输入全部从开始取)
newNodes.forEach((node) => {
node.inputs.forEach((input) => {
if (workflowStart && input.key === NodeInputKeyEnum.userChatInput) {
if (workflowStart) {
if (
node.flowNodeType === FlowNodeTypeEnum.datasetSearchNode &&
input.key === NodeInputKeyEnum.datasetSearchInput
) {
input.value = [
[workflowStart.nodeId, NodeOutputKeyEnum.userChatInput],
[workflowStart.nodeId, NodeOutputKeyEnum.userFiles]
];
input.valueType = WorkflowIOValueTypeEnum.arrayString;
return;
}
if (input.key !== NodeInputKeyEnum.userChatInput) return;
input.value = [workflowStart.nodeId, NodeOutputKeyEnum.userChatInput];
}
});
......
projects/app/src/web/core/workflow/utils.ts
......@@ -129,9 +129,15 @@ export const storeNode2FlowNode = ({
storeNode.inputs
.filter((item) => !templateInputs.find((input) => input.key === item.key))
.map((item) => {
if (!dynamicInput) return item;
const templateInput = template.inputs.find((input) => input.key === item.key);
if (!dynamicInput) {
return {
...item,
deprecated: templateInput?.deprecated
};
}
return {
...item,
...getInputComponentProps(dynamicInput),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment