Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit 84130da8 by Finley Ge Committed by GitHub
Options

fix(permission): clean dangling resource permissions (#7442)

parent 199c0dab
Showing with 891 additions and 4 deletions
packages/global/support/permission/dataClean/controller.schema.ts 0 → 100644
import z from 'zod';
import { ObjectIdSchema } from '../../../common/type/mongo';
import { BoolSchema, IntSchema } from '../../../common/zod';
export const DEFAULT_DANGLING_PERMISSION_BATCH_SIZE = 500;
export const DEFAULT_DANGLING_PERMISSION_MAX_SCAN = 10000;
export const DEFAULT_DANGLING_PERMISSION_SAMPLE_LIMIT = 20;
export const DanglingReferenceReasonSchema = z.enum([
'missingTeam',
'missingTeamMember',
'missingGroup',
'missingOrg',
'missingApp',
'missingDataset',
'missingAgentSkill',
'missingResourceId'
]);
export type DanglingReferenceReason = z.infer<typeof DanglingReferenceReasonSchema>;
export const CleanupDanglingResourcePermissionsOptionsSchema = z.object({
dryRun: z.boolean(),
batchSize: z.number().int().min(1).max(5000),
maxScan: z.number().int().min(1).max(100000),
sampleLimit: z.number().int().min(0).max(100),
cursor: ObjectIdSchema.optional()
});
export type CleanupDanglingResourcePermissionsOptions = z.infer<
typeof CleanupDanglingResourcePermissionsOptionsSchema
>;
export const CleanupDanglingResourcePermissionsBodySchema = z
.object({
dryRun: BoolSchema.optional().meta({
example: true,
description: '是否只扫描统计不删除，默认为 true'
}),
dryrun: BoolSchema.optional().meta({
example: true,
description: '是否只扫描统计不删除，兼容小写参数'
}),
batchSize: IntSchema.min(1).max(5000).optional().meta({
example: DEFAULT_DANGLING_PERMISSION_BATCH_SIZE,
description: '每批扫描的权限记录数，范围 1~5000'
}),
maxScan: IntSchema.min(1).max(100000).optional().meta({
example: DEFAULT_DANGLING_PERMISSION_MAX_SCAN,
description: '单次请求最多扫描的权限记录数，范围 1~100000'
}),
sampleLimit: IntSchema.min(0).max(100).optional().meta({
example: DEFAULT_DANGLING_PERMISSION_SAMPLE_LIMIT,
description: '返回的悬垂权限样本数，范围 0~100'
}),
cursor: ObjectIdSchema.optional().meta({
description: '上一次响应返回的 nextCursor，用于继续扫描'
})
})
.transform((body) =>
CleanupDanglingResourcePermissionsOptionsSchema.parse({
dryRun: body.dryRun ?? body.dryrun ?? true,
batchSize: body.batchSize ?? DEFAULT_DANGLING_PERMISSION_BATCH_SIZE,
maxScan: body.maxScan ?? DEFAULT_DANGLING_PERMISSION_MAX_SCAN,
sampleLimit: body.sampleLimit ?? DEFAULT_DANGLING_PERMISSION_SAMPLE_LIMIT,
cursor: body.cursor
})
);
export const DanglingPermissionSampleSchema = z.object({
permissionId: z.string().meta({ description: '悬垂权限记录 ID' }),
teamId: z.string().meta({ description: '权限记录中的团队 ID' }),
resourceType: z.string().meta({ description: '权限资源类型' }),
resourceId: z.string().optional().meta({ description: '权限资源 ID' }),
danglingReferences: z
.array(DanglingReferenceReasonSchema)
.meta({ description: '该权限记录命中的悬垂引用类型' })
});
export const DanglingReferenceReasonCountsSchema = z.object({
missingTeam: z.number().int().nonnegative().meta({ description: '团队引用缺失数量' }),
missingTeamMember: z.number().int().nonnegative().meta({ description: '成员引用缺失数量' }),
missingGroup: z.number().int().nonnegative().meta({ description: '成员组引用缺失数量' }),
missingOrg: z.number().int().nonnegative().meta({ description: '组织引用缺失数量' }),
missingApp: z.number().int().nonnegative().meta({ description: '应用引用缺失数量' }),
missingDataset: z.number().int().nonnegative().meta({ description: '知识库引用缺失数量' }),
missingAgentSkill: z.number().int().nonnegative().meta({ description: '技能引用缺失数量' }),
missingResourceId: z.number().int().nonnegative().meta({ description: '资源 ID 缺失数量' })
});
export const CleanupDanglingResourcePermissionsResponseSchema = z.object({
dryRun: z.boolean().meta({ description: '是否 dry-run' }),
scannedPermissionCount: z.number().int().nonnegative().meta({ description: '扫描权限记录数' }),
danglingPermissionCount: z
.number()
.int()
.nonnegative()
.meta({ description: '存在至少一个悬垂引用的权限记录数' }),
deletedPermissionCount: z
.number()
.int()
.nonnegative()
.meta({ description: '实际删除的权限记录数，dry-run 时为 0' }),
reasonCounts: DanglingReferenceReasonCountsSchema.meta({
description: '按悬垂引用类型统计的命中数，同一权限可能命中多种类型'
}),
batchSize: z.number().int().positive().meta({ description: '扫描批大小' }),
maxScan: z.number().int().positive().meta({ description: '单次扫描数量上限' }),
sampleLimit: z.number().int().nonnegative().meta({ description: '返回样本数量限制' }),
nextCursor: z.string().optional().meta({ description: '继续扫描时使用的游标' }),
samples: z.array(DanglingPermissionSampleSchema).meta({ description: '悬垂权限样本' })
});
export type CleanupDanglingResourcePermissionsResult = z.infer<
typeof CleanupDanglingResourcePermissionsResponseSchema
>;
packages/service/core/dataset/delete/processor.ts
......@@ -8,6 +8,8 @@ import { MongoDataset } from '../schema';
import { removeImageByPath } from '../../../common/file/image/controller';
import { MongoDatasetTraining } from '../training/schema';
import { getLogger, LogCategories } from '../../../common/logger';
import { MongoResourcePermission } from '../../../support/permission/schema';
import { PerResourceTypeEnum } from '@fastgpt/global/support/permission/constant';
const logger = getLogger(LogCategories.MODULE.DATASET.COLLECTION);
......@@ -99,11 +101,24 @@ const deleteDatasets = async ({
datasets,
session
});
});
// delete dataset
await MongoDataset.deleteMany({
_id: { $in: datasetIds }
// 权限与知识库本体同步删除，避免留下无法回收的孤立权限记录。
await MongoResourcePermission.deleteMany(
{
teamId,
resourceType: PerResourceTypeEnum.dataset,
resourceId: { $in: datasetIds }
},
{ session }
);
await MongoDataset.deleteMany(
{
teamId,
_id: { $in: datasetIds }
},
{ session }
);
});
};
......
packages/service/support/permission/dataClean/danglingPermission.ts 0 → 100644
import { PerResourceTypeEnum } from '@fastgpt/global/support/permission/constant';
import {
CleanupDanglingResourcePermissionsResponseSchema,
type CleanupDanglingResourcePermissionsOptions,
type CleanupDanglingResourcePermissionsResult,
type DanglingReferenceReason
} from '@fastgpt/global/support/permission/dataClean/controller.schema';
import { Types } from '../../../common/mongo';
import { MongoAgentSkills } from '../../../core/ai/skill/model/schema';
import { MongoApp } from '../../../core/app/schema';
import { MongoDataset } from '../../../core/dataset/schema';
import { MongoMemberGroupModel } from '../memberGroup/memberGroupSchema';
import { MongoOrgModel } from '../org/orgSchema';
import { MongoResourcePermission } from '../schema';
import { MongoTeamMember } from '../../user/team/teamMemberSchema';
import { MongoTeam } from '../../user/team/teamSchema';
type PermissionReferenceDoc = {
_id: Types.ObjectId;
teamId?: unknown;
tmbId?: unknown;
groupId?: unknown;
orgId?: unknown;
resourceType: string;
resourceId?: unknown;
};
type TeamScopedReferenceDoc = {
_id: unknown;
teamId: unknown;
};
type DanglingPermission = {
permission: PermissionReferenceDoc;
reasons: DanglingReferenceReason[];
};
const permissionSnapshotFields = [
'teamId',
'tmbId',
'groupId',
'orgId',
'resourceType',
'resourceId'
] as const;
const hasOwnField = (document: object, field: PropertyKey) =>
Object.prototype.hasOwnProperty.call(document, field);
const stringifyId = (value: unknown) => {
if (value == null) return '';
if (typeof value === 'object' && 'toString' in value && typeof value.toString === 'function') {
return value.toString();
}
return String(value);
};
const compactUniqueIds = (values: unknown[]) => {
const idMap = new Map<string, unknown>();
for (const value of values) {
const id = stringifyId(value);
if (id && Types.ObjectId.isValid(id)) idMap.set(id, value);
}
return Array.from(idMap.values());
};
const toTeamScopedReferenceKey = (id: unknown, teamId: unknown) =>
`${stringifyId(id)}:${stringifyId(teamId)}`;
const toTeamScopedReferenceSet = (documents: TeamScopedReferenceDoc[]) =>
new Set(documents.map((document) => toTeamScopedReferenceKey(document._id, document.teamId)));
const createReasonCounts = (): Record<DanglingReferenceReason, number> => ({
missingTeam: 0,
missingTeamMember: 0,
missingGroup: 0,
missingOrg: 0,
missingApp: 0,
missingDataset: 0,
missingAgentSkill: 0,
missingResourceId: 0
});
const resourceReferenceConfigs = [
{
resourceType: PerResourceTypeEnum.app,
missingReason: 'missingApp' as const,
findExisting: (ids: unknown[]) =>
MongoApp.find({ _id: { $in: ids } }, '_id teamId').lean<TeamScopedReferenceDoc[]>()
},
{
resourceType: PerResourceTypeEnum.dataset,
missingReason: 'missingDataset' as const,
findExisting: (ids: unknown[]) =>
MongoDataset.find({ _id: { $in: ids } }, '_id teamId').lean<TeamScopedReferenceDoc[]>()
},
{
resourceType: PerResourceTypeEnum.agentSkill,
missingReason: 'missingAgentSkill' as const,
findExisting: (ids: unknown[]) =>
MongoAgentSkills.find({ _id: { $in: ids } }, '_id teamId').lean<TeamScopedReferenceDoc[]>()
}
] as const;
/**
* 校验一批权限记录的外部引用。
*
* 协作者和资源引用同时校验 `_id` 与 `teamId`，跨团队引用与非法 ObjectId 均视为悬垂。
* `team` 和 `model` 权限没有 `resourceId`；其余资源类型缺少 `resourceId` 时单独报告。
*/
async function findDanglingPermissionsInBatch(
permissions: PermissionReferenceDoc[]
): Promise<DanglingPermission[]> {
const teamIds = compactUniqueIds(permissions.map((permission) => permission.teamId));
const tmbIds = compactUniqueIds(permissions.map((permission) => permission.tmbId));
const groupIds = compactUniqueIds(permissions.map((permission) => permission.groupId));
const orgIds = compactUniqueIds(permissions.map((permission) => permission.orgId));
const [teams, teamMembers, groups, orgs, resourceReferences] = await Promise.all([
MongoTeam.find({ _id: { $in: teamIds } }, '_id').lean(),
MongoTeamMember.find({ _id: { $in: tmbIds } }, '_id teamId').lean<TeamScopedReferenceDoc[]>(),
MongoMemberGroupModel.find({ _id: { $in: groupIds } }, '_id teamId').lean<
TeamScopedReferenceDoc[]
>(),
MongoOrgModel.find({ _id: { $in: orgIds } }, '_id teamId').lean<TeamScopedReferenceDoc[]>(),
Promise.all(
resourceReferenceConfigs.map(async (config) => {
const ids = compactUniqueIds(
permissions
.filter((permission) => permission.resourceType === config.resourceType)
.map((permission) => permission.resourceId)
);
const documents = await config.findExisting(ids);
return [config.resourceType, toTeamScopedReferenceSet(documents)] as const;
})
)
]);
const existingTeamIds = new Set(teams.map((team) => stringifyId(team._id)));
const existingSubjectReferenceSets = {
tmbId: toTeamScopedReferenceSet(teamMembers),
groupId: toTeamScopedReferenceSet(groups),
orgId: toTeamScopedReferenceSet(orgs)
};
const subjectReasonMap = {
tmbId: 'missingTeamMember',
groupId: 'missingGroup',
orgId: 'missingOrg'
} as const;
const existingResourceReferenceSets = new Map(resourceReferences);
return permissions.flatMap((permission) => {
const reasons: DanglingReferenceReason[] = [];
const teamId = stringifyId(permission.teamId);
if (!existingTeamIds.has(teamId)) reasons.push('missingTeam');
for (const field of Object.keys(subjectReasonMap) as (keyof typeof subjectReasonMap)[]) {
if (
hasOwnField(permission, field) &&
!existingSubjectReferenceSets[field].has(
toTeamScopedReferenceKey(permission[field], permission.teamId)
)
) {
reasons.push(subjectReasonMap[field]);
}
}
const resourceConfig = resourceReferenceConfigs.find(
(config) => config.resourceType === permission.resourceType
);
if (resourceConfig) {
const resourceId = stringifyId(permission.resourceId);
if (!resourceId) {
reasons.push('missingResourceId');
} else if (
!existingResourceReferenceSets
.get(resourceConfig.resourceType)
?.has(toTeamScopedReferenceKey(permission.resourceId, permission.teamId))
) {
reasons.push(resourceConfig.missingReason);
}
}
return reasons.length > 0 ? [{ permission, reasons }] : [];
});
}
/** 生成包含字段存在性和值的删除条件，避免扫描后的并发更新被误删。 */
const createPermissionSnapshotFilter = (permission: PermissionReferenceDoc) => ({
$and: [
{ _id: permission._id },
...permissionSnapshotFields.flatMap((field) =>
hasOwnField(permission, field)
? [{ [field]: permission[field] }, { [field]: { $exists: true } }]
: [{ [field]: { $exists: false } }]
)
]
});
/**
* 分批扫描并清理 `resource_permissions` 中的悬垂引用。
*
* `cursor` 和 `maxScan` 为单次运行提供边界；apply 模式会在删除前重新校验，并仅删除仍与
* 扫描快照一致的记录。返回 `nextCursor` 时，下一次调用应原样传回以继续扫描。
*/
export async function cleanupDanglingResourcePermissions(
options: CleanupDanglingResourcePermissionsOptions
): Promise<CleanupDanglingResourcePermissionsResult> {
let scannedPermissionCount = 0;
let danglingPermissionCount = 0;
let deletedPermissionCount = 0;
let lastScannedId: Types.ObjectId | undefined;
const reasonCounts = createReasonCounts();
const samples: CleanupDanglingResourcePermissionsResult['samples'] = [];
const processBatch = async (permissions: PermissionReferenceDoc[]) => {
scannedPermissionCount += permissions.length;
const danglingPermissions = await findDanglingPermissionsInBatch(permissions);
danglingPermissionCount += danglingPermissions.length;
for (const { permission, reasons } of danglingPermissions) {
for (const reason of reasons) reasonCounts[reason] += 1;
if (samples.length >= options.sampleLimit) continue;
samples.push({
permissionId: stringifyId(permission._id),
teamId: stringifyId(permission.teamId),
resourceType: permission.resourceType,
...(hasOwnField(permission, 'resourceId')
? { resourceId: stringifyId(permission.resourceId) }
: {}),
danglingReferences: reasons
});
}
if (!options.dryRun && danglingPermissions.length > 0) {
const revalidatedPermissions = await findDanglingPermissionsInBatch(
danglingPermissions.map(({ permission }) => permission)
);
if (revalidatedPermissions.length > 0) {
const result = await MongoResourcePermission.collection.deleteMany({
$or: revalidatedPermissions.map(({ permission }) =>
createPermissionSnapshotFilter(permission)
)
});
deletedPermissionCount += result.deletedCount;
}
}
};
const query = options.cursor ? { _id: { $gt: new Types.ObjectId(options.cursor) } } : {};
const permissionCursor = MongoResourcePermission.collection
.find<PermissionReferenceDoc>(query, {
projection: {
_id: 1,
teamId: 1,
tmbId: 1,
groupId: 1,
orgId: 1,
resourceType: 1,
resourceId: 1
}
})
.sort({ _id: 1 })
.limit(options.maxScan)
.batchSize(options.batchSize);
let batch: PermissionReferenceDoc[] = [];
for await (const permission of permissionCursor) {
batch.push(permission);
lastScannedId = permission._id;
if (batch.length < options.batchSize) continue;
await processBatch(batch);
batch = [];
}
if (batch.length > 0) await processBatch(batch);
const hasMore = lastScannedId
? Boolean(
await MongoResourcePermission.collection.findOne(
{ _id: { $gt: lastScannedId } },
{ projection: { _id: 1 } }
)
)
: false;
return CleanupDanglingResourcePermissionsResponseSchema.parse({
dryRun: options.dryRun,
scannedPermissionCount,
danglingPermissionCount,
deletedPermissionCount,
reasonCounts,
batchSize: options.batchSize,
maxScan: options.maxScan,
sampleLimit: options.sampleLimit,
...(hasMore ? { nextCursor: stringifyId(lastScannedId) } : {}),
samples
});
}
packages/service/test/core/dataset/delete/processor.test.ts 0 → 100644
import { describe, expect, it, vi } from 'vitest';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
import {
OwnerRoleVal,
PerResourceTypeEnum,
ReadRoleVal
} from '@fastgpt/global/support/permission/constant';
import { datasetDeleteProcessor } from '@fastgpt/service/core/dataset/delete/processor';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { MongoResourcePermission } from '@fastgpt/service/support/permission/schema';
import { getUser } from '@test/datas/users';
vi.mock('@fastgpt/service/common/s3/sources/dataset', () => ({
getS3DatasetSource: () => ({
deleteDatasetFilesByPrefix: vi.fn()
})
}));
describe('datasetDeleteProcessor', () => {
it('deletes permissions for the dataset and all its children', async () => {
const user = await getUser('dataset-delete-permission');
const otherTeamUser = await getUser('dataset-delete-permission-other-team');
const deleteTime = new Date();
const rootDataset = await MongoDataset.create({
teamId: user.teamId,
tmbId: user.tmbId,
name: 'root folder',
type: DatasetTypeEnum.folder,
deleteTime
});
const childDataset = await MongoDataset.create({
teamId: user.teamId,
tmbId: user.tmbId,
parentId: rootDataset._id,
name: 'child dataset',
type: DatasetTypeEnum.dataset,
deleteTime
});
const retainedDataset = await MongoDataset.create({
teamId: user.teamId,
tmbId: user.tmbId,
name: 'retained dataset',
type: DatasetTypeEnum.dataset
});
await MongoResourcePermission.insertMany([
{
teamId: user.teamId,
tmbId: user.tmbId,
resourceType: PerResourceTypeEnum.dataset,
resourceId: rootDataset._id,
permission: OwnerRoleVal
},
{
teamId: user.teamId,
tmbId: user.tmbId,
resourceType: PerResourceTypeEnum.dataset,
resourceId: childDataset._id,
permission: ReadRoleVal
},
{
teamId: user.teamId,
tmbId: user.tmbId,
resourceType: PerResourceTypeEnum.dataset,
resourceId: retainedDataset._id,
permission: OwnerRoleVal
},
{
teamId: user.teamId,
tmbId: user.tmbId,
resourceType: PerResourceTypeEnum.app,
resourceId: rootDataset._id,
permission: OwnerRoleVal
},
{
teamId: otherTeamUser.teamId,
tmbId: otherTeamUser.tmbId,
resourceType: PerResourceTypeEnum.dataset,
resourceId: rootDataset._id,
permission: OwnerRoleVal
}
]);
await datasetDeleteProcessor({
data: {
teamId: user.teamId,
datasetId: String(rootDataset._id)
}
} as never);
expect(
await MongoResourcePermission.countDocuments({
teamId: user.teamId,
resourceType: PerResourceTypeEnum.dataset,
resourceId: { $in: [rootDataset._id, childDataset._id] }
})
).toBe(0);
expect(
await MongoResourcePermission.countDocuments({
teamId: user.teamId,
resourceType: PerResourceTypeEnum.dataset,
resourceId: retainedDataset._id
})
).toBe(1);
expect(
await MongoResourcePermission.countDocuments({
teamId: user.teamId,
resourceType: PerResourceTypeEnum.app,
resourceId: rootDataset._id
})
).toBe(1);
expect(
await MongoResourcePermission.countDocuments({
teamId: otherTeamUser.teamId,
resourceType: PerResourceTypeEnum.dataset,
resourceId: rootDataset._id
})
).toBe(1);
expect(
await MongoDataset.countDocuments({ _id: { $in: [rootDataset._id, childDataset._id] } })
).toBe(0);
expect(await MongoDataset.countDocuments({ _id: retainedDataset._id })).toBe(1);
});
});
packages/service/test/support/permission/dataClean/danglingPermission.test.ts 0 → 100644
import { beforeEach, describe, expect, it, vi } from 'vitest';
import { OwnerRoleVal, PerResourceTypeEnum } from '@fastgpt/global/support/permission/constant';
import { Types } from '@fastgpt/service/common/mongo';
import { MongoAgentSkills } from '@fastgpt/service/core/ai/skill/model/schema';
import { MongoApp } from '@fastgpt/service/core/app/schema';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { cleanupDanglingResourcePermissions } from '@fastgpt/service/support/permission/dataClean/danglingPermission';
import { MongoMemberGroupModel } from '@fastgpt/service/support/permission/memberGroup/memberGroupSchema';
import { MongoOrgModel } from '@fastgpt/service/support/permission/org/orgSchema';
import { MongoResourcePermission } from '@fastgpt/service/support/permission/schema';
import { getUser } from '@test/datas/users';
const objectId = () => new Types.ObjectId();
describe('cleanupDanglingResourcePermissions', () => {
let expectedDanglingPermissionIds: string[];
let validAppId: Types.ObjectId;
let concurrentlyAssignedAppId: Types.ObjectId;
beforeEach(async () => {
const user = await getUser(`permission-cleanup-${objectId()}`);
const otherTeamUser = await getUser(`permission-cleanup-other-${objectId()}`);
validAppId = objectId();
concurrentlyAssignedAppId = objectId();
const validDatasetId = objectId();
const validSkillId = objectId();
const crossTeamAppId = objectId();
const validGroupId = objectId();
const validOrgId = objectId();
await Promise.all([
MongoApp.collection.insertOne({ _id: validAppId, teamId: user.teamId }),
MongoApp.collection.insertOne({ _id: concurrentlyAssignedAppId, teamId: user.teamId }),
MongoDataset.collection.insertOne({ _id: validDatasetId, teamId: user.teamId }),
MongoAgentSkills.collection.insertOne({ _id: validSkillId, teamId: user.teamId }),
MongoApp.collection.insertOne({ _id: crossTeamAppId, teamId: otherTeamUser.teamId }),
MongoMemberGroupModel.collection.insertOne({ _id: validGroupId, teamId: user.teamId }),
MongoOrgModel.collection.insertOne({ _id: validOrgId, teamId: user.teamId })
]);
const createPermission = ({
teamId = user.teamId,
tmbId,
groupId,
orgId,
resourceType,
resourceId,
resourceName
}: {
teamId?: string;
tmbId?: string;
groupId?: Types.ObjectId;
orgId?: Types.ObjectId;
resourceType: PerResourceTypeEnum;
resourceId?: unknown;
resourceName?: string;
}) => ({
_id: objectId(),
teamId: new Types.ObjectId(teamId),
...(tmbId !== undefined ? { tmbId: new Types.ObjectId(tmbId) } : {}),
...(groupId !== undefined ? { groupId } : {}),
...(orgId !== undefined ? { orgId } : {}),
resourceType,
...(resourceId !== undefined ? { resourceId } : {}),
...(resourceName ? { resourceName } : {}),
permission: OwnerRoleVal
});
const validPermissions = [
createPermission({ tmbId: user.tmbId, resourceType: PerResourceTypeEnum.team }),
createPermission({
tmbId: user.tmbId,
resourceType: PerResourceTypeEnum.model,
resourceName: 'gpt-4o'
}),
createPermission({
tmbId: user.tmbId,
resourceType: PerResourceTypeEnum.app,
resourceId: validAppId
}),
createPermission({
groupId: validGroupId,
resourceType: PerResourceTypeEnum.dataset,
resourceId: validDatasetId
}),
createPermission({
orgId: validOrgId,
resourceType: PerResourceTypeEnum.agentSkill,
resourceId: validSkillId
})
];
const danglingPermissions = [
createPermission({ teamId: String(objectId()), resourceType: PerResourceTypeEnum.team }),
createPermission({ tmbId: String(objectId()), resourceType: PerResourceTypeEnum.team }),
createPermission({ groupId: objectId(), resourceType: PerResourceTypeEnum.team }),
createPermission({ orgId: objectId(), resourceType: PerResourceTypeEnum.team }),
createPermission({
tmbId: user.tmbId,
resourceType: PerResourceTypeEnum.app,
resourceId: objectId()
}),
createPermission({
tmbId: user.tmbId,
resourceType: PerResourceTypeEnum.dataset,
resourceId: objectId()
}),
createPermission({
tmbId: user.tmbId,
resourceType: PerResourceTypeEnum.agentSkill,
resourceId: objectId()
}),
createPermission({ tmbId: user.tmbId, resourceType: PerResourceTypeEnum.app }),
createPermission({ tmbId: otherTeamUser.tmbId, resourceType: PerResourceTypeEnum.team }),
createPermission({
tmbId: user.tmbId,
resourceType: PerResourceTypeEnum.app,
resourceId: crossTeamAppId
})
];
const malformedPermission = {
_id: objectId(),
teamId: 'invalid-team-id',
tmbId: 'invalid-team-member-id',
groupId: 'invalid-group-id',
orgId: 'invalid-org-id',
resourceType: PerResourceTypeEnum.app,
resourceId: 'invalid-app-id',
permission: OwnerRoleVal
};
const malformedFalsyPermission = {
_id: objectId(),
teamId: new Types.ObjectId(user.teamId),
tmbId: '',
groupId: 0,
orgId: null,
resourceType: PerResourceTypeEnum.app,
resourceId: '',
permission: OwnerRoleVal
};
expectedDanglingPermissionIds = [
...danglingPermissions,
malformedPermission,
malformedFalsyPermission
].map((permission) => String(permission._id));
await MongoResourcePermission.collection.insertMany([
...validPermissions,
...danglingPermissions,
malformedPermission,
malformedFalsyPermission
]);
});
it('reports every dangling reference without deleting permissions during dry-run', async () => {
const result = await cleanupDanglingResourcePermissions({
dryRun: true,
batchSize: 2,
maxScan: 100,
sampleLimit: 20
});
expect(result).toMatchObject({
dryRun: true,
scannedPermissionCount: 17,
danglingPermissionCount: 12,
deletedPermissionCount: 0,
reasonCounts: {
missingTeam: 2,
missingTeamMember: 4,
missingGroup: 3,
missingOrg: 3,
missingApp: 3,
missingDataset: 1,
missingAgentSkill: 1,
missingResourceId: 2
}
});
expect(result.samples.map((sample) => sample.permissionId).sort()).toEqual(
expectedDanglingPermissionIds.sort()
);
expect(await MongoResourcePermission.countDocuments()).toBe(17);
});
it('deletes only dangling permissions in apply mode', async () => {
const result = await cleanupDanglingResourcePermissions({
dryRun: false,
batchSize: 3,
maxScan: 100,
sampleLimit: 2
});
expect(result).toMatchObject({
dryRun: false,
scannedPermissionCount: 17,
danglingPermissionCount: 12,
deletedPermissionCount: 12,
sampleLimit: 2
});
expect(result.samples).toHaveLength(2);
expect(
await MongoResourcePermission.countDocuments({
_id: { $in: expectedDanglingPermissionIds }
})
).toBe(0);
expect(await MongoResourcePermission.countDocuments()).toBe(5);
});
it('keeps a permission that becomes valid after validation', async () => {
const permissionId = new Types.ObjectId(expectedDanglingPermissionIds[4]);
const originalDeleteMany = MongoResourcePermission.collection.deleteMany.bind(
MongoResourcePermission.collection
);
vi.spyOn(MongoResourcePermission.collection, 'deleteMany').mockImplementationOnce(
async (filter, options) => {
await MongoResourcePermission.collection.updateOne(
{ _id: permissionId },
{ $set: { resourceId: concurrentlyAssignedAppId } }
);
return originalDeleteMany(filter, options);
}
);
const result = await cleanupDanglingResourcePermissions({
dryRun: false,
batchSize: 100,
maxScan: 100,
sampleLimit: 0
});
expect(result.deletedPermissionCount).toBe(11);
expect(await MongoResourcePermission.countDocuments({ _id: permissionId })).toBe(1);
});
it('supports bounded scans with a resumable cursor', async () => {
let cursor: string | undefined;
let scannedPermissionCount = 0;
do {
const result = await cleanupDanglingResourcePermissions({
dryRun: true,
batchSize: 2,
maxScan: 5,
sampleLimit: 0,
cursor
});
scannedPermissionCount += result.scannedPermissionCount;
cursor = result.nextCursor;
} while (cursor);
expect(scannedPermissionCount).toBe(17);
expect(await MongoResourcePermission.countDocuments()).toBe(17);
});
});
projects/app/src/pages/api/admin/dataClean/cleanupDanglingResourcePermissions.ts 0 → 100644
import { NextAPI } from '@/service/middleware/entry';
import {
CleanupDanglingResourcePermissionsBodySchema,
CleanupDanglingResourcePermissionsResponseSchema,
type CleanupDanglingResourcePermissionsResult
} from '@fastgpt/global/support/permission/dataClean/controller.schema';
import type { ApiRequestProps } from '@fastgpt/next/type';
import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError';
import { cleanupDanglingResourcePermissions } from '@fastgpt/service/support/permission/dataClean/danglingPermission';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
/* ============================================================================
* API: 清理悬垂资源权限
* Route: POST /api/admin/dataClean/cleanupDanglingResourcePermissions
* Method: POST
* Description: 检查权限记录引用的团队、协作者和资源是否存在，可选择删除悬垂权限。
* Tags: ['Admin', 'DataClean', 'Permission', 'Delete']
* ============================================================================ */
/** 管理员权限悬垂引用清理入口，默认仅执行 dry-run。 */
async function handler(req: ApiRequestProps): Promise<CleanupDanglingResourcePermissionsResult> {
await authCert({ req, authRoot: true });
const { body } = parseApiInput({
req,
bodySchema: CleanupDanglingResourcePermissionsBodySchema
});
return CleanupDanglingResourcePermissionsResponseSchema.parse(
await cleanupDanglingResourcePermissions(body)
);
}
export default NextAPI(handler);
projects/app/test/pages/api/admin/dataClean/cleanupDanglingResourcePermissions.test.ts 0 → 100644
import { beforeEach, describe, expect, it } from 'vitest';
import { OwnerRoleVal, PerResourceTypeEnum } from '@fastgpt/global/support/permission/constant';
import { Types } from '@fastgpt/service/common/mongo';
import { MongoResourcePermission } from '@fastgpt/service/support/permission/schema';
import cleanupDanglingResourcePermissionsHandler from '@/pages/api/admin/dataClean/cleanupDanglingResourcePermissions';
import { getRootUser, getUser } from '@test/datas/users';
import { Call } from '@test/utils/request';
describe('cleanupDanglingResourcePermissions data clean API', () => {
beforeEach(async () => {
const user = await getUser(`permission-cleanup-api-${new Types.ObjectId()}`);
await MongoResourcePermission.collection.insertOne({
_id: new Types.ObjectId(),
teamId: new Types.ObjectId(user.teamId),
tmbId: new Types.ObjectId(user.tmbId),
resourceType: PerResourceTypeEnum.app,
resourceId: new Types.ObjectId(),
permission: OwnerRoleVal
});
});
it('defaults to dry-run when the flag is omitted', async () => {
const rootUser = await getRootUser();
const response = await Call(cleanupDanglingResourcePermissionsHandler, {
auth: rootUser,
body: {
batchSize: 4,
maxScan: 100,
sampleLimit: 0
}
});
expect(response.error).toBeUndefined();
expect(response.data).toMatchObject({
dryRun: true,
scannedPermissionCount: 1,
danglingPermissionCount: 1,
deletedPermissionCount: 0,
samples: []
});
expect(await MongoResourcePermission.countDocuments()).toBe(1);
});
});
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment