Commit a14f76c3 by DigHuang Committed by GitHub

feat(app): introduce resource reference index and optimize skill appCount query performance (#6980)

* feat(app): introduce resource reference index and optimize skill appCount query performance

* refactor(app): standardize api input validation and optimize backfill performance

* fix: tighten skill resource refs backfill

* fix: use version resource refs for skill app count

* fix: cache skill resource refs on apps

* chore: remove unused app version refs index

* perf: optimize skill list permission lookup

---------

Co-authored-by: archer <545436317@qq.com>
parent e97332a7
......@@ -84,6 +84,11 @@ export const AppChatConfigTypeSchema = z.object({
});
export type AppChatConfigType = z.infer<typeof AppChatConfigTypeSchema>;
export const AppResourceRefsSchema = z.object({
skillIds: z.array(z.string()).default([])
});
export type AppResourceRefsType = z.infer<typeof AppResourceRefsSchema>;
// Mongo Collection
export const AppSchemaTypeSchema = z.object({
_id: ObjectIdSchema,
......@@ -115,7 +120,7 @@ export const AppSchemaTypeSchema = z.object({
chatConfig: AppChatConfigTypeSchema,
scheduledTriggerConfig: AppScheduledTriggerConfigTypeSchema.optional(),
scheduledTriggerNextTime: z.coerce.date().optional(),
resourceRefs: AppResourceRefsSchema.optional(),
inheritPermission: z.boolean().optional(),
// if access the app by favourite or quick
......
import { AppSchemaTypeSchema } from '../type';
import { AppResourceRefsSchema } from '../type';
import { SourceMemberSchema } from '../../../support/user/type';
import z from 'zod';
import { ObjectIdSchema } from '../../../common/type/mongo';
......@@ -13,7 +14,8 @@ export const AppVersionSchema = z.object({
chatConfig: AppSchemaTypeSchema.shape.chatConfig,
isPublish: z.boolean().optional(),
isAutoSave: z.boolean().optional(),
versionName: z.string()
versionName: z.string(),
resourceRefs: AppResourceRefsSchema.optional()
});
export type AppVersionSchemaType = z.infer<typeof AppVersionSchema>;
......@@ -27,3 +29,19 @@ export const VersionListItemSchema = z.object({
sourceMember: SourceMemberSchema
});
export type VersionListItemType = z.infer<typeof VersionListItemSchema>;
/* Publish app */
export const PublishAppQuerySchema = z.object({
appId: z.string()
});
export type PublishAppQueryType = z.infer<typeof PublishAppQuerySchema>;
export const PublishAppBodySchema = z.object({
nodes: AppSchemaTypeSchema.shape.modules.optional(),
edges: AppSchemaTypeSchema.shape.edges.optional(),
chatConfig: AppSchemaTypeSchema.shape.chatConfig.optional(),
isPublish: z.boolean().optional(),
versionName: z.string().optional(),
autoSave: z.boolean().optional()
});
export type PublishAppBodyType = z.infer<typeof PublishAppBodySchema>;
......@@ -66,10 +66,6 @@ export function validateSandboxConfig(config: SandboxProviderConfig): void {
throw new Error(`Invalid runtime: ${config.runtime}`);
}
if (config.provider === 'opensandbox' && !config.apiKey) {
throw new Error('Sandbox provider apiKey is required for opensandbox');
}
if (config.provider === 'sealosdevbox' && !config.token) {
throw new Error('Sandbox provider token is required for sealosdevbox');
}
......
import type { AppResourceRefsType } from '@fastgpt/global/core/app/type';
import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import type { StoreNodeItemType } from '@fastgpt/global/core/workflow/type/node';
export const AppResourceRefsSkillIdsPath = 'resourceRefs.skillIds';
/**
* 从应用 workflow 节点中提取该版本引用的资源索引。
* 当前只维护 skillIds,后续有真实消费场景时再补充其他资源字段。
* 创建、保存版本、发布和历史回填共用同一套规则,避免不同写入路径统计口径漂移。
*/
export function extractAppResourceRefsFromNodes(
nodes: StoreNodeItemType[] | null | undefined = []
): AppResourceRefsType {
const skillIds = new Set<string>();
const nodeList = Array.isArray(nodes) ? nodes : [];
nodeList.forEach((node) => {
node.inputs?.forEach((input) => {
if (input.key !== NodeInputKeyEnum.skills) return;
const value = input.value;
const skills = Array.isArray(value) ? value : value ? [value] : [];
skills.forEach((item) => {
const skillId = item?.skillId;
if (skillId) {
skillIds.add(String(skillId));
}
});
});
});
return {
skillIds: Array.from(skillIds)
};
}
const getSkillIdCondition = (skillIds: string | string[]) => {
const list = Array.isArray(skillIds) ? skillIds : [skillIds];
return list.length === 1 ? list[0] : { $in: list };
};
/**
* 构造基于 resourceRefs 的 Skill 引用查询。
* apps.resourceRefs 是最新发布版本缓存,app_versions.resourceRefs 是版本事实记录。
*/
export function buildAppSkillRefMongoQuery(skillIds: string | string[]) {
return {
[AppResourceRefsSkillIdsPath]: getSkillIdCondition(skillIds)
};
}
......@@ -104,7 +104,12 @@ const AppSchema = new Schema(
scheduledTriggerNextTime: {
type: Date
},
resourceRefs: {
skillIds: {
type: [String],
default: []
}
},
inheritPermission: {
type: Boolean,
default: true
......@@ -134,6 +139,7 @@ const AppSchema = new Schema(
AppSchema.index({ teamId: 1, updateTime: -1 });
AppSchema.index({ teamId: 1, type: 1 });
AppSchema.index({ teamId: 1, deleteTime: 1, 'resourceRefs.skillIds': 1 });
// Schedule
AppSchema.index(
......
......@@ -35,7 +35,13 @@ const AppVersionSchema = new Schema(
},
isPublish: Boolean,
isAutoSave: Boolean,
versionName: String
versionName: String,
resourceRefs: {
skillIds: {
type: [String],
default: []
}
}
},
{
minimize: false
......
......@@ -21,9 +21,7 @@ export const hasAgentSandboxConfig = () => {
}
if (provider === 'opensandbox') {
return !!(
process.env.AGENT_SANDBOX_OPENSANDBOX_BASEURL && process.env.AGENT_SANDBOX_OPENSANDBOX_API_KEY
);
return !!process.env.AGENT_SANDBOX_OPENSANDBOX_BASEURL;
}
return false;
......
......@@ -288,7 +288,7 @@ describe('sandbox provider config', () => {
).toThrow('Invalid runtime: invalid');
});
it('requires opensandbox api key for docker runtime', async () => {
it('does not require opensandbox api key for docker runtime', async () => {
const { validateSandboxConfig } = await loadSandboxConfigModule();
expect(() =>
......@@ -298,7 +298,7 @@ describe('sandbox provider config', () => {
apiKey: '',
runtime: 'docker'
})
).toThrow('Sandbox provider apiKey is required for opensandbox');
).not.toThrow();
});
it('throws for unsupported provider in config switch', async () => {
......
import { NextAPI } from '@/service/middleware/entry';
import { AppFolderTypeList } from '@fastgpt/global/core/app/constants';
import type { StoreNodeItemType } from '@fastgpt/global/core/workflow/type/node';
import { extractAppResourceRefsFromNodes } from '@fastgpt/service/core/app/resourceRefs';
import { MongoApp } from '@fastgpt/service/core/app/schema';
import { MongoAppVersion } from '@fastgpt/service/core/app/version/schema';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError';
import { getLogger } from '@fastgpt/service/common/logger';
import type { ApiRequestProps } from '@fastgpt/service/type/next';
import { Types } from '@fastgpt/service/common/mongo';
import { z } from 'zod';
const logger = getLogger(['backfillAppResourceRefs']);
const DEFAULT_BACKFILL_START_TIME = new Date('2026-05-24T00:00:00.000+08:00');
const QuerySchema = z.object({
dryRun: z
.enum(['true', 'false'])
.optional()
.default('false')
.transform((value) => value === 'true'),
batchSize: z.coerce.number().int().min(1).max(2000).optional().default(500),
startTime: z.coerce.date().optional().default(DEFAULT_BACKFILL_START_TIME)
});
type MigrationStats = {
matched: number;
updated: number;
};
type ResponseType = {
message: string;
dryRun: boolean;
batchSize: number;
startTime: string;
versions: MigrationStats;
apps: MigrationStats;
};
/**
* 回填指定时间后的 app_versions.resourceRefs,保证新版 Skill 发布后的版本记录具备统一索引。
*/
async function backfillVersionResourceRefs({
dryRun,
batchSize,
startTime
}: {
dryRun: boolean;
batchSize: number;
startTime: Date;
}): Promise<MigrationStats> {
let lastId: string | undefined;
let scanned = 0;
let matched = 0;
let updated = 0;
while (true) {
const versions = await MongoAppVersion.find(
{
...(lastId ? { _id: { $gt: new Types.ObjectId(lastId) } } : {}),
time: { $gte: startTime }
},
'_id nodes'
)
.sort({ _id: 1 })
.limit(batchSize)
.lean();
if (versions.length === 0) break;
scanned += versions.length;
lastId = String(versions[versions.length - 1]._id);
const operations = versions.map((version) => ({
updateOne: {
filter: { _id: version._id },
update: {
$set: {
resourceRefs: extractAppResourceRefsFromNodes(version.nodes)
}
}
}
}));
matched += operations.length;
if (!dryRun && operations.length > 0) {
const result = await MongoAppVersion.bulkWrite(operations, { ordered: false });
updated += result.modifiedCount + result.upsertedCount;
}
logger.info('[version resource refs] backfill progress', {
scanned,
matched,
updated: dryRun ? 0 : updated,
lastId,
startTime
});
}
return { matched, updated: dryRun ? 0 : updated };
}
/**
* 按指定时间后的最新已发布版本刷新 apps.resourceRefs 缓存。
* 草稿和自动保存不会影响缓存,避免 Skill 关联统计偏离线上发布态。
*/
async function backfillAppResourceRefs({
dryRun,
batchSize,
startTime
}: {
dryRun: boolean;
batchSize: number;
startTime: Date;
}): Promise<MigrationStats> {
let lastId: string | undefined;
let scanned = 0;
let matched = 0;
let updated = 0;
while (true) {
const apps = await MongoApp.find(
{
...(lastId ? { _id: { $gt: new Types.ObjectId(lastId) } } : {}),
type: { $nin: AppFolderTypeList },
deleteTime: null,
updateTime: { $gte: startTime }
},
'_id'
)
.sort({ _id: 1 })
.limit(batchSize)
.lean();
if (apps.length === 0) break;
scanned += apps.length;
lastId = String(apps[apps.length - 1]._id);
const latestPublishedVersions = await MongoAppVersion.aggregate<{
_id: Types.ObjectId;
nodes?: StoreNodeItemType[];
}>([
{
$match: {
appId: { $in: apps.map((app) => new Types.ObjectId(String(app._id))) },
isPublish: true,
time: { $gte: startTime }
}
},
{
$sort: {
appId: 1,
time: -1,
_id: -1
}
},
{
$group: {
_id: '$appId',
nodes: { $first: '$nodes' }
}
}
]);
const operations = latestPublishedVersions.map((version) => ({
updateOne: {
filter: { _id: version._id },
update: {
$set: {
resourceRefs: extractAppResourceRefsFromNodes(version.nodes)
}
}
}
}));
matched += operations.length;
if (!dryRun && operations.length > 0) {
const result = await MongoApp.bulkWrite(operations, { ordered: false });
updated += result.modifiedCount + result.upsertedCount;
}
logger.info('[app resource refs] backfill progress', {
scanned,
matched,
updated: dryRun ? 0 : updated,
lastId,
startTime
});
}
return { matched, updated: dryRun ? 0 : updated };
}
async function handler(
req: ApiRequestProps<undefined, z.input<typeof QuerySchema>>
): Promise<ResponseType> {
await authCert({ req, authRoot: true });
const { dryRun, batchSize, startTime } = parseApiInput({
req,
querySchema: QuerySchema
}).query;
logger.info('Start app resource refs backfill', { dryRun, batchSize, startTime });
const versions = await backfillVersionResourceRefs({ dryRun, batchSize, startTime });
const apps = await backfillAppResourceRefs({ dryRun, batchSize, startTime });
logger.info('Finish app resource refs backfill', {
dryRun,
batchSize,
startTime,
versions,
apps
});
return {
message: 'Completed app resource refs backfill',
dryRun,
batchSize,
startTime: startTime.toISOString(),
versions,
apps
};
}
export default NextAPI(handler);
......@@ -12,7 +12,6 @@ import { authUserPer } from '@fastgpt/service/support/permission/user/auth';
import { getGroupsByTmbId } from '@fastgpt/service/support/permission/memberGroup/controllers';
import { getOrgIdSetWithParentByTmbId } from '@fastgpt/service/support/permission/org/controllers';
import { addSourceMember } from '@fastgpt/service/support/user/utils';
import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import { sumPer } from '@fastgpt/global/support/permission/utils';
import type {
ListAppsBySkillIdQuery,
......@@ -20,6 +19,7 @@ import type {
} from '@fastgpt/global/core/ai/skill/api';
import { ListAppsBySkillIdQuerySchema } from '@fastgpt/global/core/ai/skill/api';
import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError';
import { buildAppSkillRefMongoQuery } from '@fastgpt/service/core/app/resourceRefs';
async function handler(
req: ApiRequestProps<unknown, ListAppsBySkillIdQuery>
......@@ -60,21 +60,12 @@ async function handler(
myOrgSet.has(String(item.orgId))
);
// Query apps whose modules contain the given skillId
// 查询最新发布版本缓存引用该 skillId 的应用。
const apps = await MongoApp.find(
{
teamId,
deleteTime: null,
modules: {
$elemMatch: {
inputs: {
$elemMatch: {
key: NodeInputKeyEnum.skills,
'value.skillId': skillId
}
}
}
}
...buildAppSkillRefMongoQuery(skillId)
},
'_id parentId avatar type name intro tmbId updateTime inheritPermission'
)
......@@ -108,8 +99,8 @@ async function handler(
return {
_id: String(app._id),
name: app.name,
avatar: app.avatar,
intro: app.intro,
avatar: app.avatar || '',
intro: app.intro || '',
tmbId: String(app.tmbId),
type: app.type,
updateTime: app.updateTime,
......
......@@ -9,9 +9,10 @@ import { GetSkillDetailQuerySchema } from '@fastgpt/global/core/ai/skill/api';
import { isValidObjectId } from 'mongoose';
import type { ApiRequestProps } from '@fastgpt/service/type/next';
import { MongoApp } from '@fastgpt/service/core/app/schema';
import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import { Types } from '@fastgpt/service/common/mongo';
import { SkillErrEnum } from '@fastgpt/global/common/error/code/skill';
import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError';
import { buildAppSkillRefMongoQuery } from '@fastgpt/service/core/app/resourceRefs';
async function handler(
req: ApiRequestProps<Record<string, never>, GetSkillDetailQuery>
......@@ -22,7 +23,7 @@ async function handler(
return Promise.reject(SkillErrEnum.invalidSkillId);
}
const { skill, permission } = await authSkill({
const { skill, permission, teamId } = await authSkill({
req,
authToken: true,
authApiKey: true,
......@@ -31,17 +32,9 @@ async function handler(
});
const appCount = await MongoApp.countDocuments({
teamId: new Types.ObjectId(String(teamId)),
deleteTime: null,
modules: {
$elemMatch: {
inputs: {
$elemMatch: {
key: NodeInputKeyEnum.skills,
'value.skillId': skill._id.toString()
}
}
}
}
...buildAppSkillRefMongoQuery(skill._id.toString())
});
return {
......
import { NextAPI } from '@/service/middleware/entry';
import { MongoAgentSkills } from '@fastgpt/service/core/ai/skill/model/schema';
import { MongoApp } from '@fastgpt/service/core/app/schema';
import { Types } from '@fastgpt/service/common/mongo';
import { authUserPer } from '@fastgpt/service/support/permission/user/auth';
import { SkillPermission } from '@fastgpt/global/support/permission/skill/controller';
import {
......@@ -17,9 +18,12 @@ import { getOrgIdSetWithParentByTmbId } from '@fastgpt/service/support/permissio
import { addSourceMember } from '@fastgpt/service/support/user/utils';
import { sumPer } from '@fastgpt/global/support/permission/utils';
import { AgentSkillTypeEnum, AgentSkillSourceEnum } from '@fastgpt/global/core/ai/skill/constants';
import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import { ListSkillsQuerySchema, type ListSkillsQuery } from '@fastgpt/global/core/ai/skill/api';
import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError';
import {
AppResourceRefsSkillIdsPath,
buildAppSkillRefMongoQuery
} from '@fastgpt/service/core/app/resourceRefs';
export type GetSkillListBody = ListSkillsQuery;
......@@ -93,6 +97,34 @@ async function handler(req: ApiRequestProps<GetSkillListBody>) {
myGroupMap.has(String(item.groupId)) ||
myOrgSet.has(String(item.orgId))
);
const myRoleResourceIds = Array.from(
new Map(myRoles.map((item) => [String(item.resourceId), item.resourceId])).values()
);
const roleCountByResourceId = new Map<string, number>();
roleList.forEach((item) => {
const resourceId = String(item.resourceId);
roleCountByResourceId.set(resourceId, (roleCountByResourceId.get(resourceId) ?? 0) + 1);
});
const myTmbRoleByResourceId = new Map<string, ReturnType<typeof sumPer>>();
const myGroupOrgRoleListByResourceId = new Map<string, Parameters<typeof sumPer>>();
myRoles.forEach((item) => {
const resourceId = String(item.resourceId);
if (item.tmbId) {
myTmbRoleByResourceId.set(resourceId, item.permission);
return;
}
if (item.groupId || item.orgId) {
const permissionList = myGroupOrgRoleListByResourceId.get(resourceId) ?? [];
permissionList.push(item.permission);
myGroupOrgRoleListByResourceId.set(resourceId, permissionList);
}
});
const myGroupOrgRoleByResourceId = new Map<string, ReturnType<typeof sumPer>>();
myGroupOrgRoleListByResourceId.forEach((permissionList, resourceId) => {
myGroupOrgRoleByResourceId.set(resourceId, sumPer(...permissionList));
});
const findSkillQuery = (() => {
const sourceQuery = (() => {
......@@ -134,7 +166,7 @@ async function handler(req: ApiRequestProps<GetSkillListBody>) {
}
// Filter skills by permission, if not owner, only get skills that I have permission to access
const idList = { _id: { $in: myRoles.map((item) => item.resourceId) } };
const idList = { _id: { $in: myRoleResourceIds } };
const skillPerQuery = teamPer.isOwner
? {}
: parentId
......@@ -168,23 +200,15 @@ async function handler(req: ApiRequestProps<GetSkillListBody>) {
.map((skill) => {
const { Per, privateSkill } = (() => {
const getPer = (skillId: string) => {
const tmbRole = myRoles.find(
(item) => String(item.resourceId) === skillId && !!item.tmbId
)?.permission;
const groupAndOrgRole = sumPer(
...myRoles
.filter(
(item) => String(item.resourceId) === skillId && (!!item.groupId || !!item.orgId)
)
.map((item) => item.permission)
);
const tmbRole = myTmbRoleByResourceId.get(skillId);
const groupAndOrgRole = myGroupOrgRoleByResourceId.get(skillId);
return new SkillPermission({
role: tmbRole ?? groupAndOrgRole,
isOwner: String(skill.tmbId) === String(tmbId) || teamPer.isOwner
});
};
const getClbCount = (skillId: string) => {
return roleList.filter((item) => String(item.resourceId) === String(skillId)).length;
return roleCountByResourceId.get(skillId) ?? 0;
};
// inherit
......@@ -221,33 +245,6 @@ async function handler(req: ApiRequestProps<GetSkillListBody>) {
})
.filter((skill) => skill.permission.hasReadPer);
// 默认保持历史行为返回 appCount;编辑页状态校验可显式关闭,避免批量校验时产生额外 count 查询。
const nonFolderSkills =
withAppCount !== false ? formatSkills.filter((s) => s.type !== AgentSkillTypeEnum.folder) : [];
const appCountMap = new Map<string, number>();
if (nonFolderSkills.length > 0) {
const counts = await Promise.all(
nonFolderSkills.map((skill) =>
MongoApp.countDocuments({
deleteTime: null,
modules: {
$elemMatch: {
inputs: {
$elemMatch: {
key: NodeInputKeyEnum.skills,
'value.skillId': skill._id.toString()
}
}
}
}
})
)
);
nonFolderSkills.forEach((skill, i) => {
appCountMap.set(skill._id.toString(), counts[i]);
});
}
const total = formatSkills.length;
// Apply pagination if requested
......@@ -259,6 +256,34 @@ async function handler(req: ApiRequestProps<GetSkillListBody>) {
return formatSkills;
})();
// 默认保持历史行为返回 appCount;只统计本次返回的 skill,编辑页状态校验可显式关闭。
const nonFolderSkills =
withAppCount !== false ? pagedSkills.filter((s) => s.type !== AgentSkillTypeEnum.folder) : [];
const appCountMap = new Map<string, number>();
if (nonFolderSkills.length > 0) {
const skillIdStrings = nonFolderSkills.map((skill) => String(skill._id));
const counts = await MongoApp.aggregate<{ _id: string; count: number }>([
{
$match: {
teamId: new Types.ObjectId(String(teamId)),
deleteTime: null,
...buildAppSkillRefMongoQuery(skillIdStrings)
}
},
{ $unwind: `$${AppResourceRefsSkillIdsPath}` },
{ $match: buildAppSkillRefMongoQuery(skillIdStrings) },
{
$group: {
_id: `$${AppResourceRefsSkillIdsPath}`,
count: { $sum: 1 }
}
}
]);
counts.forEach((item) => {
appCountMap.set(String(item._id), item.count);
});
}
const listWithAppCount = pagedSkills.map((skill) => ({
...skill,
appCount: appCountMap.get(skill._id.toString()) ?? 0
......
......@@ -36,10 +36,15 @@ import { isS3ObjectKey } from '@fastgpt/service/common/s3/utils';
import { MongoAppTemplate } from '@fastgpt/service/core/app/templates/templateSchema';
import { updateParentFoldersUpdateTime } from '@fastgpt/service/core/app/controller';
import { copyAvatarImage } from '@fastgpt/service/common/file/image/controller';
import { extractAppResourceRefsFromNodes } from '@fastgpt/service/core/app/resourceRefs';
import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError';
async function handler(req: ApiRequestProps<CreateAppBodyType>) {
const parseResult = await CreateAppBodySchema.safeParseAsync(req.body);
const body = parseResult.success ? parseResult.data : req.body;
const { body } = parseApiInput({
req,
bodySchema: CreateAppBodySchema
});
const { parentId, name, avatar, intro, type, modules, edges, chatConfig, templateId, utmParams } =
body;
......@@ -152,6 +157,7 @@ export const onCreateApp = async ({
}
const create = async (session: ClientSession) => {
const resourceRefs = extractAppResourceRefsFromNodes(modules);
const _avatar = await (async () => {
if (!templateId) return avatar;
......@@ -186,7 +192,8 @@ export const onCreateApp = async ({
type,
version: 'v2',
pluginData,
templateId
templateId,
...(!AppFolderTypeList.includes(type!) && { resourceRefs })
}
],
{ session, ordered: true }
......@@ -206,7 +213,8 @@ export const onCreateApp = async ({
versionName: name,
username,
avatar: userAvatar,
isPublish: true
isPublish: true,
resourceRefs
}
],
{ session, ordered: true }
......
import type { NextApiResponse } from 'next';
import { NextAPI } from '@/service/middleware/entry';
import { authApp } from '@fastgpt/service/support/permission/app/auth';
import { MongoAppVersion } from '@fastgpt/service/core/app/version/schema';
......@@ -14,10 +13,19 @@ import { AuditEventEnum } from '@fastgpt/global/support/user/audit/constants';
import { getI18nAppType } from '@fastgpt/service/support/user/audit/util';
import { i18nT } from '@fastgpt/global/common/i18n/utils';
import { updateParentFoldersUpdateTime } from '@fastgpt/service/core/app/controller';
import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError';
import { extractAppResourceRefsFromNodes } from '@fastgpt/service/core/app/resourceRefs';
import { PublishAppQuerySchema, PublishAppBodySchema } from '@fastgpt/global/core/app/version/type';
async function handler(req: ApiRequestProps<PostPublishAppProps>, res: NextApiResponse<any>) {
const { appId } = req.query as { appId: string };
const { nodes = [], edges = [], chatConfig, isPublish, versionName, autoSave } = req.body;
async function handler(req: ApiRequestProps<PostPublishAppProps>) {
const {
query: { appId },
body: { nodes = [], edges = [], chatConfig, isPublish, versionName, autoSave }
} = parseApiInput({
req,
querySchema: PublishAppQuerySchema,
bodySchema: PublishAppBodySchema
});
const { app, tmbId, teamId } = await authApp({
appId,
......@@ -29,6 +37,7 @@ async function handler(req: ApiRequestProps<PostPublishAppProps>, res: NextApiRe
beforeUpdateAppFormat({
nodes
});
const resourceRefs = extractAppResourceRefsFromNodes(nodes);
updateParentFoldersUpdateTime({
parentId: app.parentId
});
......@@ -47,7 +56,8 @@ async function handler(req: ApiRequestProps<PostPublishAppProps>, res: NextApiRe
edges,
chatConfig,
versionName: i18nT('app:auto_save'),
time: new Date()
time: new Date(),
resourceRefs
},
{ session, upsert: true }
......@@ -93,34 +103,38 @@ async function handler(req: ApiRequestProps<PostPublishAppProps>, res: NextApiRe
chatConfig,
isPublish,
versionName,
tmbId
tmbId,
resourceRefs
}
],
{ session, ordered: true }
);
// update app
const setUpdate = {
modules: nodes,
edges,
chatConfig,
updateTime: new Date(),
version: 'v2',
...(isPublish && { resourceRefs }),
...(isPublish && chatConfig?.scheduledTriggerConfig?.cronString
? {
scheduledTriggerConfig: chatConfig.scheduledTriggerConfig,
scheduledTriggerNextTime: getNextTimeByCronStringAndTimezone(
chatConfig.scheduledTriggerConfig
)
}
: {}),
'pluginData.nodeVersion': _id
};
await MongoApp.updateOne(
{ _id: appId },
{
modules: nodes,
edges,
chatConfig,
updateTime: new Date(),
version: 'v2',
// 只有发布才会更新定时器
...(isPublish &&
(chatConfig?.scheduledTriggerConfig?.cronString
? {
$set: {
scheduledTriggerConfig: chatConfig.scheduledTriggerConfig,
scheduledTriggerNextTime: getNextTimeByCronStringAndTimezone(
chatConfig.scheduledTriggerConfig
)
}
}
: { $unset: { scheduledTriggerConfig: '', scheduledTriggerNextTime: '' } })),
'pluginData.nodeVersion': _id
$set: setUpdate,
...(isPublish && !chatConfig?.scheduledTriggerConfig?.cronString
? { $unset: { scheduledTriggerConfig: '', scheduledTriggerNextTime: '' } }
: {})
},
{
session
......
import { describe, expect, it } from 'vitest';
import handler from '@/pages/api/admin/backfillAppResourceRefs';
import { AppTypeEnum } from '@fastgpt/global/core/app/constants';
import { NodeInputKeyEnum, WorkflowIOValueTypeEnum } from '@fastgpt/global/core/workflow/constants';
import type { StoreNodeItemType } from '@fastgpt/global/core/workflow/type/node';
import { FlowNodeInputTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
import { getNanoid } from '@fastgpt/global/common/string/tools';
import { MongoApp } from '@fastgpt/service/core/app/schema';
import { MongoAppVersion } from '@fastgpt/service/core/app/version/schema';
import { getUser } from '@test/datas/users';
import { Call } from '@test/utils/request';
const startTime = new Date('2026-05-24T00:00:00.000+08:00');
const beforeStartTime = new Date('2026-05-23T23:59:59.999+08:00');
const afterStartTime = new Date('2026-05-24T12:00:00.000+08:00');
const createSkillNode = (skillId: string): StoreNodeItemType =>
({
nodeId: `node-${skillId}`,
name: 'Agent',
inputs: [
{
key: NodeInputKeyEnum.skills,
renderTypeList: [FlowNodeInputTypeEnum.selectSkill],
valueType: WorkflowIOValueTypeEnum.arrayObject,
value: [{ skillId }]
}
],
outputs: []
}) as StoreNodeItemType;
describe('GET /api/admin/backfillAppResourceRefs', () => {
it('只回填 2026-05-24 之后的版本资源索引和最新发布应用缓存', async () => {
const user = await getUser(`backfill-app-resource-refs-${getNanoid(6)}`);
const oldSkillId = `old-skill-${getNanoid(6)}`;
const newSkillId = `new-skill-${getNanoid(6)}`;
const draftSkillId = `draft-skill-${getNanoid(6)}`;
const [oldApp, newApp, draftOnlyApp] = await MongoApp.create([
{
name: 'Old App',
type: AppTypeEnum.chatAgent,
teamId: user.teamId,
tmbId: user.tmbId,
modules: [createSkillNode(oldSkillId)],
edges: [],
chatConfig: {},
updateTime: afterStartTime,
resourceRefs: { skillIds: ['stale-old-app'] }
},
{
name: 'New App',
type: AppTypeEnum.chatAgent,
teamId: user.teamId,
tmbId: user.tmbId,
modules: [createSkillNode(newSkillId)],
edges: [],
chatConfig: {},
updateTime: afterStartTime,
resourceRefs: { skillIds: ['stale-new-app'] }
},
{
name: 'Draft Only App',
type: AppTypeEnum.chatAgent,
teamId: user.teamId,
tmbId: user.tmbId,
modules: [createSkillNode(draftSkillId)],
edges: [],
chatConfig: {},
updateTime: afterStartTime,
resourceRefs: { skillIds: ['stale-draft-app'] }
}
]);
const [oldVersion, newVersion, draftVersion] = await MongoAppVersion.create([
{
appId: oldApp._id,
tmbId: user.tmbId,
nodes: [createSkillNode(oldSkillId)],
edges: [],
chatConfig: {},
isPublish: true,
versionName: 'old publish',
time: beforeStartTime,
resourceRefs: { skillIds: ['stale-old-version'] }
},
{
appId: newApp._id,
tmbId: user.tmbId,
nodes: [createSkillNode(newSkillId)],
edges: [],
chatConfig: {},
isPublish: true,
versionName: 'new publish',
time: afterStartTime,
resourceRefs: { skillIds: ['stale-new-version'] }
},
{
appId: draftOnlyApp._id,
tmbId: user.tmbId,
nodes: [createSkillNode(draftSkillId)],
edges: [],
chatConfig: {},
isPublish: false,
versionName: 'draft save',
time: afterStartTime,
resourceRefs: { skillIds: ['stale-draft-version'] }
}
]);
const res = await Call(handler, {
auth: { ...user, isRoot: true },
query: {
dryRun: 'false',
batchSize: '1'
}
});
expect(res.code).toBe(200);
expect(res.data.startTime).toBe(startTime.toISOString());
expect(res.data.versions.matched).toBe(2);
expect(res.data.apps.matched).toBe(1);
const [updatedOldVersion, updatedNewVersion, updatedDraftVersion] = await Promise.all([
MongoAppVersion.findById(oldVersion._id).lean(),
MongoAppVersion.findById(newVersion._id).lean(),
MongoAppVersion.findById(draftVersion._id).lean()
]);
expect(updatedOldVersion?.resourceRefs?.skillIds).toEqual(['stale-old-version']);
expect(updatedNewVersion?.resourceRefs?.skillIds).toEqual([newSkillId]);
expect(updatedDraftVersion?.resourceRefs?.skillIds).toEqual([draftSkillId]);
const [updatedOldApp, updatedNewApp, updatedDraftOnlyApp] = await Promise.all([
MongoApp.findById(oldApp._id).lean(),
MongoApp.findById(newApp._id).lean(),
MongoApp.findById(draftOnlyApp._id).lean()
]);
expect(updatedOldApp?.resourceRefs?.skillIds).toEqual(['stale-old-app']);
expect(updatedNewApp?.resourceRefs?.skillIds).toEqual([newSkillId]);
expect(updatedDraftOnlyApp?.resourceRefs?.skillIds).toEqual(['stale-draft-app']);
});
});
import { describe, expect, it } from 'vitest';
import handler from '@/pages/api/core/ai/skill/list';
import publishHandler from '@/pages/api/core/app/version/publish';
import { MongoAgentSkills } from '@fastgpt/service/core/ai/skill/model/schema';
import { MongoApp } from '@fastgpt/service/core/app/schema';
import { MongoAppVersion } from '@fastgpt/service/core/app/version/schema';
import { AgentSkillSourceEnum, AgentSkillTypeEnum } from '@fastgpt/global/core/ai/skill/constants';
import { getNanoid } from '@fastgpt/global/common/string/tools';
import { getUser } from '@test/datas/users';
import { Call } from '@test/utils/request';
import type { ListSkillsQuery, ListSkillsResponse } from '@fastgpt/global/core/ai/skill/api';
import { onCreateApp } from '@/pages/api/core/app/create';
import { AppTypeEnum } from '@fastgpt/global/core/app/constants';
import {
FlowNodeInputTypeEnum,
FlowNodeTypeEnum
} from '@fastgpt/global/core/workflow/node/constant';
import { NodeInputKeyEnum, WorkflowIOValueTypeEnum } from '@fastgpt/global/core/workflow/constants';
import type { StoreNodeItemType } from '@fastgpt/global/core/workflow/type/node';
describe('POST /api/core/ai/skill/list', () => {
it('按 skillIds 查询时不受父目录过滤影响,并排除已删除 Skill', async () => {
......@@ -50,4 +61,144 @@ describe('POST /api/core/ai/skill/list', () => {
expect(res.code).toBe(200);
expect(res.data.list.map((item) => String(item._id))).toEqual([String(activeSkill._id)]);
});
it('appCount 基于已发布版本的 resourceRefs,草稿保存不影响统计', async () => {
const user = await getUser(`agent-skill-list-published-refs-${getNanoid(6)}`);
const [publishedSkill, draftSkill] = await MongoAgentSkills.create([
{
name: 'Published Skill',
type: AgentSkillTypeEnum.skill,
source: AgentSkillSourceEnum.personal,
teamId: user.teamId,
tmbId: user.tmbId
},
{
name: 'Draft Skill',
type: AgentSkillTypeEnum.skill,
source: AgentSkillSourceEnum.personal,
teamId: user.teamId,
tmbId: user.tmbId
}
]);
const createSkillNode = (skillId: string): StoreNodeItemType =>
({
nodeId: `node-${skillId}`,
name: 'Agent',
flowNodeType: FlowNodeTypeEnum.agent,
inputs: [
{
key: NodeInputKeyEnum.skills,
label: 'Skills',
renderTypeList: [FlowNodeInputTypeEnum.selectSkill],
valueType: WorkflowIOValueTypeEnum.arrayObject,
value: [{ skillId }]
}
],
outputs: []
}) as StoreNodeItemType;
const appId = await onCreateApp({
name: 'Skill Ref App',
type: AppTypeEnum.chatAgent,
modules: [createSkillNode(String(publishedSkill._id))],
edges: [],
chatConfig: {},
teamId: user.teamId,
tmbId: user.tmbId
});
await expect(MongoApp.findById(appId).lean()).resolves.toMatchObject({
resourceRefs: { skillIds: [String(publishedSkill._id)] }
});
const draftSaveRes = await Call(publishHandler, {
auth: user,
query: { appId },
body: {
nodes: [createSkillNode(String(draftSkill._id))],
edges: [],
chatConfig: {},
isPublish: false,
versionName: 'draft'
}
});
expect(draftSaveRes.code).toBe(200);
await expect(MongoApp.findById(appId).lean()).resolves.toMatchObject({
resourceRefs: { skillIds: [String(publishedSkill._id)] }
});
const draftRes = await Call<ListSkillsQuery, Record<string, never>, ListSkillsResponse>(
handler,
{
auth: user,
body: {
source: 'mine',
parentId: null
}
}
);
expect(draftRes.code).toBe(200);
const getCount = (skillId: string) =>
draftRes.data.list.find((item) => String(item._id) === skillId)?.appCount;
expect(getCount(String(publishedSkill._id))).toBe(1);
expect(getCount(String(draftSkill._id))).toBe(0);
const publishRes = await Call(publishHandler, {
auth: user,
query: { appId },
body: {
nodes: [createSkillNode(String(draftSkill._id))],
edges: [],
chatConfig: {},
isPublish: true,
versionName: 'publish draft'
}
});
expect(publishRes.code).toBe(200);
await expect(MongoApp.findById(appId).lean()).resolves.toMatchObject({
resourceRefs: { skillIds: [String(draftSkill._id)] }
});
const publishedRes = await Call<ListSkillsQuery, Record<string, never>, ListSkillsResponse>(
handler,
{
auth: user,
body: {
source: 'mine',
parentId: null
}
}
);
const getPublishedCount = (skillId: string) =>
publishedRes.data.list.find((item) => String(item._id) === skillId)?.appCount;
expect(getPublishedCount(String(publishedSkill._id))).toBe(0);
expect(getPublishedCount(String(draftSkill._id))).toBe(1);
const latestPublishedVersion = await MongoAppVersion.findOne({
appId,
isPublish: true
})
.sort({ time: -1, _id: -1 })
.lean();
expect(latestPublishedVersion?.resourceRefs?.skillIds).toEqual([String(draftSkill._id)]);
await MongoApp.updateOne({ _id: appId }, { $set: { resourceRefs: { skillIds: [] } } });
const clearedAppRefsRes = await Call<
ListSkillsQuery,
Record<string, never>,
ListSkillsResponse
>(handler, {
auth: user,
body: {
source: 'mine',
parentId: null
}
});
const getClearedAppRefsCount = (skillId: string) =>
clearedAppRefsRes.data.list.find((item) => String(item._id) === skillId)?.appCount;
expect(getClearedAppRefsCount(String(publishedSkill._id))).toBe(0);
expect(getClearedAppRefsCount(String(draftSkill._id))).toBe(0);
});
});
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment