Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit 84130da8 by Finley Ge Committed by GitHub
Options

fix(permission): clean dangling resource permissions (#7442)

parent 199c0dab
Showing with 590 additions and 4 deletions
packages/global/support/permission/dataClean/controller.schema.ts 0 → 100644
import z from 'zod';
import { ObjectIdSchema } from '../../../common/type/mongo';
import { BoolSchema, IntSchema } from '../../../common/zod';
export const DEFAULT_DANGLING_PERMISSION_BATCH_SIZE = 500;
export const DEFAULT_DANGLING_PERMISSION_MAX_SCAN = 10000;
export const DEFAULT_DANGLING_PERMISSION_SAMPLE_LIMIT = 20;
export const DanglingReferenceReasonSchema = z.enum([
'missingTeam',
'missingTeamMember',
'missingGroup',
'missingOrg',
'missingApp',
'missingDataset',
'missingAgentSkill',
'missingResourceId'
]);
export type DanglingReferenceReason = z.infer<typeof DanglingReferenceReasonSchema>;
export const CleanupDanglingResourcePermissionsOptionsSchema = z.object({
dryRun: z.boolean(),
batchSize: z.number().int().min(1).max(5000),
maxScan: z.number().int().min(1).max(100000),
sampleLimit: z.number().int().min(0).max(100),
cursor: ObjectIdSchema.optional()
});
export type CleanupDanglingResourcePermissionsOptions = z.infer<
typeof CleanupDanglingResourcePermissionsOptionsSchema
>;
export const CleanupDanglingResourcePermissionsBodySchema = z
.object({
dryRun: BoolSchema.optional().meta({
example: true,
description: '是否只扫描统计不删除，默认为 true'
}),
dryrun: BoolSchema.optional().meta({
example: true,
description: '是否只扫描统计不删除，兼容小写参数'
}),
batchSize: IntSchema.min(1).max(5000).optional().meta({
example: DEFAULT_DANGLING_PERMISSION_BATCH_SIZE,
description: '每批扫描的权限记录数，范围 1~5000'
}),
maxScan: IntSchema.min(1).max(100000).optional().meta({
example: DEFAULT_DANGLING_PERMISSION_MAX_SCAN,
description: '单次请求最多扫描的权限记录数，范围 1~100000'
}),
sampleLimit: IntSchema.min(0).max(100).optional().meta({
example: DEFAULT_DANGLING_PERMISSION_SAMPLE_LIMIT,
description: '返回的悬垂权限样本数，范围 0~100'
}),
cursor: ObjectIdSchema.optional().meta({
description: '上一次响应返回的 nextCursor，用于继续扫描'
})
})
.transform((body) =>
CleanupDanglingResourcePermissionsOptionsSchema.parse({
dryRun: body.dryRun ?? body.dryrun ?? true,
batchSize: body.batchSize ?? DEFAULT_DANGLING_PERMISSION_BATCH_SIZE,
maxScan: body.maxScan ?? DEFAULT_DANGLING_PERMISSION_MAX_SCAN,
sampleLimit: body.sampleLimit ?? DEFAULT_DANGLING_PERMISSION_SAMPLE_LIMIT,
cursor: body.cursor
})
);
export const DanglingPermissionSampleSchema = z.object({
permissionId: z.string().meta({ description: '悬垂权限记录 ID' }),
teamId: z.string().meta({ description: '权限记录中的团队 ID' }),
resourceType: z.string().meta({ description: '权限资源类型' }),
resourceId: z.string().optional().meta({ description: '权限资源 ID' }),
danglingReferences: z
.array(DanglingReferenceReasonSchema)
.meta({ description: '该权限记录命中的悬垂引用类型' })
});
export const DanglingReferenceReasonCountsSchema = z.object({
missingTeam: z.number().int().nonnegative().meta({ description: '团队引用缺失数量' }),
missingTeamMember: z.number().int().nonnegative().meta({ description: '成员引用缺失数量' }),
missingGroup: z.number().int().nonnegative().meta({ description: '成员组引用缺失数量' }),
missingOrg: z.number().int().nonnegative().meta({ description: '组织引用缺失数量' }),
missingApp: z.number().int().nonnegative().meta({ description: '应用引用缺失数量' }),
missingDataset: z.number().int().nonnegative().meta({ description: '知识库引用缺失数量' }),
missingAgentSkill: z.number().int().nonnegative().meta({ description: '技能引用缺失数量' }),
missingResourceId: z.number().int().nonnegative().meta({ description: '资源 ID 缺失数量' })
});
export const CleanupDanglingResourcePermissionsResponseSchema = z.object({
dryRun: z.boolean().meta({ description: '是否 dry-run' }),
scannedPermissionCount: z.number().int().nonnegative().meta({ description: '扫描权限记录数' }),
danglingPermissionCount: z
.number()
.int()
.nonnegative()
.meta({ description: '存在至少一个悬垂引用的权限记录数' }),
deletedPermissionCount: z
.number()
.int()
.nonnegative()
.meta({ description: '实际删除的权限记录数，dry-run 时为 0' }),
reasonCounts: DanglingReferenceReasonCountsSchema.meta({
description: '按悬垂引用类型统计的命中数，同一权限可能命中多种类型'
}),
batchSize: z.number().int().positive().meta({ description: '扫描批大小' }),
maxScan: z.number().int().positive().meta({ description: '单次扫描数量上限' }),
sampleLimit: z.number().int().nonnegative().meta({ description: '返回样本数量限制' }),
nextCursor: z.string().optional().meta({ description: '继续扫描时使用的游标' }),
samples: z.array(DanglingPermissionSampleSchema).meta({ description: '悬垂权限样本' })
});
export type CleanupDanglingResourcePermissionsResult = z.infer<
typeof CleanupDanglingResourcePermissionsResponseSchema
>;
packages/service/core/dataset/delete/processor.ts
...@@ -8,6 +8,8 @@ import { MongoDataset } from '../schema'; ...@@ -8,6 +8,8 @@ import { MongoDataset } from '../schema';
import { removeImageByPath } from '../../../common/file/image/controller'; import { removeImageByPath } from '../../../common/file/image/controller';
import { MongoDatasetTraining } from '../training/schema'; import { MongoDatasetTraining } from '../training/schema';
import { getLogger, LogCategories } from '../../../common/logger'; import { getLogger, LogCategories } from '../../../common/logger';
import { MongoResourcePermission } from '../../../support/permission/schema';
import { PerResourceTypeEnum } from '@fastgpt/global/support/permission/constant';
const logger = getLogger(LogCategories.MODULE.DATASET.COLLECTION); const logger = getLogger(LogCategories.MODULE.DATASET.COLLECTION);
...@@ -99,11 +101,24 @@ const deleteDatasets = async ({ ...@@ -99,11 +101,24 @@ const deleteDatasets = async ({
datasets, datasets,
session session
}); });
});
// delete dataset // 权限与知识库本体同步删除，避免留下无法回收的孤立权限记录。
await MongoDataset.deleteMany({ await MongoResourcePermission.deleteMany(
_id: { $in: datasetIds } {
teamId,
resourceType: PerResourceTypeEnum.dataset,
resourceId: { $in: datasetIds }
},
{ session }
);
await MongoDataset.deleteMany(
{
teamId,
_id: { $in: datasetIds }
},
{ session }
);
}); });
}; };
......
packages/service/test/core/dataset/delete/processor.test.ts 0 → 100644
import { describe, expect, it, vi } from 'vitest';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
import {
OwnerRoleVal,
PerResourceTypeEnum,
ReadRoleVal
} from '@fastgpt/global/support/permission/constant';
import { datasetDeleteProcessor } from '@fastgpt/service/core/dataset/delete/processor';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { MongoResourcePermission } from '@fastgpt/service/support/permission/schema';
import { getUser } from '@test/datas/users';
vi.mock('@fastgpt/service/common/s3/sources/dataset', () => ({
getS3DatasetSource: () => ({
deleteDatasetFilesByPrefix: vi.fn()
})
}));
describe('datasetDeleteProcessor', () => {
it('deletes permissions for the dataset and all its children', async () => {
const user = await getUser('dataset-delete-permission');
const otherTeamUser = await getUser('dataset-delete-permission-other-team');
const deleteTime = new Date();
const rootDataset = await MongoDataset.create({
teamId: user.teamId,
tmbId: user.tmbId,
name: 'root folder',
type: DatasetTypeEnum.folder,
deleteTime
});
const childDataset = await MongoDataset.create({
teamId: user.teamId,
tmbId: user.tmbId,
parentId: rootDataset._id,
name: 'child dataset',
type: DatasetTypeEnum.dataset,
deleteTime
});
const retainedDataset = await MongoDataset.create({
teamId: user.teamId,
tmbId: user.tmbId,
name: 'retained dataset',
type: DatasetTypeEnum.dataset
});
await MongoResourcePermission.insertMany([
{
teamId: user.teamId,
tmbId: user.tmbId,
resourceType: PerResourceTypeEnum.dataset,
resourceId: rootDataset._id,
permission: OwnerRoleVal
},
{
teamId: user.teamId,
tmbId: user.tmbId,
resourceType: PerResourceTypeEnum.dataset,
resourceId: childDataset._id,
permission: ReadRoleVal
},
{
teamId: user.teamId,
tmbId: user.tmbId,
resourceType: PerResourceTypeEnum.dataset,
resourceId: retainedDataset._id,
permission: OwnerRoleVal
},
{
teamId: user.teamId,
tmbId: user.tmbId,
resourceType: PerResourceTypeEnum.app,
resourceId: rootDataset._id,
permission: OwnerRoleVal
},
{
teamId: otherTeamUser.teamId,
tmbId: otherTeamUser.tmbId,
resourceType: PerResourceTypeEnum.dataset,
resourceId: rootDataset._id,
permission: OwnerRoleVal
}
]);
await datasetDeleteProcessor({
data: {
teamId: user.teamId,
datasetId: String(rootDataset._id)
}
} as never);
expect(
await MongoResourcePermission.countDocuments({
teamId: user.teamId,
resourceType: PerResourceTypeEnum.dataset,
resourceId: { $in: [rootDataset._id, childDataset._id] }
})
).toBe(0);
expect(
await MongoResourcePermission.countDocuments({
teamId: user.teamId,
resourceType: PerResourceTypeEnum.dataset,
resourceId: retainedDataset._id
})
).toBe(1);
expect(
await MongoResourcePermission.countDocuments({
teamId: user.teamId,
resourceType: PerResourceTypeEnum.app,
resourceId: rootDataset._id
})
).toBe(1);
expect(
await MongoResourcePermission.countDocuments({
teamId: otherTeamUser.teamId,
resourceType: PerResourceTypeEnum.dataset,
resourceId: rootDataset._id
})
).toBe(1);
expect(
await MongoDataset.countDocuments({ _id: { $in: [rootDataset._id, childDataset._id] } })
).toBe(0);
expect(await MongoDataset.countDocuments({ _id: retainedDataset._id })).toBe(1);
});
});
packages/service/test/support/permission/dataClean/danglingPermission.test.ts 0 → 100644
import { beforeEach, describe, expect, it, vi } from 'vitest';
import { OwnerRoleVal, PerResourceTypeEnum } from '@fastgpt/global/support/permission/constant';
import { Types } from '@fastgpt/service/common/mongo';
import { MongoAgentSkills } from '@fastgpt/service/core/ai/skill/model/schema';
import { MongoApp } from '@fastgpt/service/core/app/schema';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { cleanupDanglingResourcePermissions } from '@fastgpt/service/support/permission/dataClean/danglingPermission';
import { MongoMemberGroupModel } from '@fastgpt/service/support/permission/memberGroup/memberGroupSchema';
import { MongoOrgModel } from '@fastgpt/service/support/permission/org/orgSchema';
import { MongoResourcePermission } from '@fastgpt/service/support/permission/schema';
import { getUser } from '@test/datas/users';
const objectId = () => new Types.ObjectId();
describe('cleanupDanglingResourcePermissions', () => {
let expectedDanglingPermissionIds: string[];
let validAppId: Types.ObjectId;
let concurrentlyAssignedAppId: Types.ObjectId;
beforeEach(async () => {
const user = await getUser(`permission-cleanup-${objectId()}`);
const otherTeamUser = await getUser(`permission-cleanup-other-${objectId()}`);
validAppId = objectId();
concurrentlyAssignedAppId = objectId();
const validDatasetId = objectId();
const validSkillId = objectId();
const crossTeamAppId = objectId();
const validGroupId = objectId();
const validOrgId = objectId();
await Promise.all([
MongoApp.collection.insertOne({ _id: validAppId, teamId: user.teamId }),
MongoApp.collection.insertOne({ _id: concurrentlyAssignedAppId, teamId: user.teamId }),
MongoDataset.collection.insertOne({ _id: validDatasetId, teamId: user.teamId }),
MongoAgentSkills.collection.insertOne({ _id: validSkillId, teamId: user.teamId }),
MongoApp.collection.insertOne({ _id: crossTeamAppId, teamId: otherTeamUser.teamId }),
MongoMemberGroupModel.collection.insertOne({ _id: validGroupId, teamId: user.teamId }),
MongoOrgModel.collection.insertOne({ _id: validOrgId, teamId: user.teamId })
]);
const createPermission = ({
teamId = user.teamId,
tmbId,
groupId,
orgId,
resourceType,
resourceId,
resourceName
}: {
teamId?: string;
tmbId?: string;
groupId?: Types.ObjectId;
orgId?: Types.ObjectId;
resourceType: PerResourceTypeEnum;
resourceId?: unknown;
resourceName?: string;
}) => ({
_id: objectId(),
teamId: new Types.ObjectId(teamId),
...(tmbId !== undefined ? { tmbId: new Types.ObjectId(tmbId) } : {}),
...(groupId !== undefined ? { groupId } : {}),
...(orgId !== undefined ? { orgId } : {}),
resourceType,
...(resourceId !== undefined ? { resourceId } : {}),
...(resourceName ? { resourceName } : {}),
permission: OwnerRoleVal
});
const validPermissions = [
createPermission({ tmbId: user.tmbId, resourceType: PerResourceTypeEnum.team }),
createPermission({
tmbId: user.tmbId,
resourceType: PerResourceTypeEnum.model,
resourceName: 'gpt-4o'
}),
createPermission({
tmbId: user.tmbId,
resourceType: PerResourceTypeEnum.app,
resourceId: validAppId
}),
createPermission({
groupId: validGroupId,
resourceType: PerResourceTypeEnum.dataset,
resourceId: validDatasetId
}),
createPermission({
orgId: validOrgId,
resourceType: PerResourceTypeEnum.agentSkill,
resourceId: validSkillId
})
];
const danglingPermissions = [
createPermission({ teamId: String(objectId()), resourceType: PerResourceTypeEnum.team }),
createPermission({ tmbId: String(objectId()), resourceType: PerResourceTypeEnum.team }),
createPermission({ groupId: objectId(), resourceType: PerResourceTypeEnum.team }),
createPermission({ orgId: objectId(), resourceType: PerResourceTypeEnum.team }),
createPermission({
tmbId: user.tmbId,
resourceType: PerResourceTypeEnum.app,
resourceId: objectId()
}),
createPermission({
tmbId: user.tmbId,
resourceType: PerResourceTypeEnum.dataset,
resourceId: objectId()
}),
createPermission({
tmbId: user.tmbId,
resourceType: PerResourceTypeEnum.agentSkill,
resourceId: objectId()
}),
createPermission({ tmbId: user.tmbId, resourceType: PerResourceTypeEnum.app }),
createPermission({ tmbId: otherTeamUser.tmbId, resourceType: PerResourceTypeEnum.team }),
createPermission({
tmbId: user.tmbId,
resourceType: PerResourceTypeEnum.app,
resourceId: crossTeamAppId
})
];
const malformedPermission = {
_id: objectId(),
teamId: 'invalid-team-id',
tmbId: 'invalid-team-member-id',
groupId: 'invalid-group-id',
orgId: 'invalid-org-id',
resourceType: PerResourceTypeEnum.app,
resourceId: 'invalid-app-id',
permission: OwnerRoleVal
};
const malformedFalsyPermission = {
_id: objectId(),
teamId: new Types.ObjectId(user.teamId),
tmbId: '',
groupId: 0,
orgId: null,
resourceType: PerResourceTypeEnum.app,
resourceId: '',
permission: OwnerRoleVal
};
expectedDanglingPermissionIds = [
...danglingPermissions,
malformedPermission,
malformedFalsyPermission
].map((permission) => String(permission._id));
await MongoResourcePermission.collection.insertMany([
...validPermissions,
...danglingPermissions,
malformedPermission,
malformedFalsyPermission
]);
});
it('reports every dangling reference without deleting permissions during dry-run', async () => {
const result = await cleanupDanglingResourcePermissions({
dryRun: true,
batchSize: 2,
maxScan: 100,
sampleLimit: 20
});
expect(result).toMatchObject({
dryRun: true,
scannedPermissionCount: 17,
danglingPermissionCount: 12,
deletedPermissionCount: 0,
reasonCounts: {
missingTeam: 2,
missingTeamMember: 4,
missingGroup: 3,
missingOrg: 3,
missingApp: 3,
missingDataset: 1,
missingAgentSkill: 1,
missingResourceId: 2
}
});
expect(result.samples.map((sample) => sample.permissionId).sort()).toEqual(
expectedDanglingPermissionIds.sort()
);
expect(await MongoResourcePermission.countDocuments()).toBe(17);
});
it('deletes only dangling permissions in apply mode', async () => {
const result = await cleanupDanglingResourcePermissions({
dryRun: false,
batchSize: 3,
maxScan: 100,
sampleLimit: 2
});
expect(result).toMatchObject({
dryRun: false,
scannedPermissionCount: 17,
danglingPermissionCount: 12,
deletedPermissionCount: 12,
sampleLimit: 2
});
expect(result.samples).toHaveLength(2);
expect(
await MongoResourcePermission.countDocuments({
_id: { $in: expectedDanglingPermissionIds }
})
).toBe(0);
expect(await MongoResourcePermission.countDocuments()).toBe(5);
});
it('keeps a permission that becomes valid after validation', async () => {
const permissionId = new Types.ObjectId(expectedDanglingPermissionIds[4]);
const originalDeleteMany = MongoResourcePermission.collection.deleteMany.bind(
MongoResourcePermission.collection
);
vi.spyOn(MongoResourcePermission.collection, 'deleteMany').mockImplementationOnce(
async (filter, options) => {
await MongoResourcePermission.collection.updateOne(
{ _id: permissionId },
{ $set: { resourceId: concurrentlyAssignedAppId } }
);
return originalDeleteMany(filter, options);
}
);
const result = await cleanupDanglingResourcePermissions({
dryRun: false,
batchSize: 100,
maxScan: 100,
sampleLimit: 0
});
expect(result.deletedPermissionCount).toBe(11);
expect(await MongoResourcePermission.countDocuments({ _id: permissionId })).toBe(1);
});
it('supports bounded scans with a resumable cursor', async () => {
let cursor: string | undefined;
let scannedPermissionCount = 0;
do {
const result = await cleanupDanglingResourcePermissions({
dryRun: true,
batchSize: 2,
maxScan: 5,
sampleLimit: 0,
cursor
});
scannedPermissionCount += result.scannedPermissionCount;
cursor = result.nextCursor;
} while (cursor);
expect(scannedPermissionCount).toBe(17);
expect(await MongoResourcePermission.countDocuments()).toBe(17);
});
});
projects/app/src/pages/api/admin/dataClean/cleanupDanglingResourcePermissions.ts 0 → 100644
import { NextAPI } from '@/service/middleware/entry';
import {
CleanupDanglingResourcePermissionsBodySchema,
CleanupDanglingResourcePermissionsResponseSchema,
type CleanupDanglingResourcePermissionsResult
} from '@fastgpt/global/support/permission/dataClean/controller.schema';
import type { ApiRequestProps } from '@fastgpt/next/type';
import { parseApiInput } from '@fastgpt/service/common/zod/requestParseError';
import { cleanupDanglingResourcePermissions } from '@fastgpt/service/support/permission/dataClean/danglingPermission';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
/* ============================================================================
* API: 清理悬垂资源权限
* Route: POST /api/admin/dataClean/cleanupDanglingResourcePermissions
* Method: POST
* Description: 检查权限记录引用的团队、协作者和资源是否存在，可选择删除悬垂权限。
* Tags: ['Admin', 'DataClean', 'Permission', 'Delete']
* ============================================================================ */
/** 管理员权限悬垂引用清理入口，默认仅执行 dry-run。 */
async function handler(req: ApiRequestProps): Promise<CleanupDanglingResourcePermissionsResult> {
await authCert({ req, authRoot: true });
const { body } = parseApiInput({
req,
bodySchema: CleanupDanglingResourcePermissionsBodySchema
});
return CleanupDanglingResourcePermissionsResponseSchema.parse(
await cleanupDanglingResourcePermissions(body)
);
}
export default NextAPI(handler);
projects/app/test/pages/api/admin/dataClean/cleanupDanglingResourcePermissions.test.ts 0 → 100644
import { beforeEach, describe, expect, it } from 'vitest';
import { OwnerRoleVal, PerResourceTypeEnum } from '@fastgpt/global/support/permission/constant';
import { Types } from '@fastgpt/service/common/mongo';
import { MongoResourcePermission } from '@fastgpt/service/support/permission/schema';
import cleanupDanglingResourcePermissionsHandler from '@/pages/api/admin/dataClean/cleanupDanglingResourcePermissions';
import { getRootUser, getUser } from '@test/datas/users';
import { Call } from '@test/utils/request';
describe('cleanupDanglingResourcePermissions data clean API', () => {
beforeEach(async () => {
const user = await getUser(`permission-cleanup-api-${new Types.ObjectId()}`);
await MongoResourcePermission.collection.insertOne({
_id: new Types.ObjectId(),
teamId: new Types.ObjectId(user.teamId),
tmbId: new Types.ObjectId(user.tmbId),
resourceType: PerResourceTypeEnum.app,
resourceId: new Types.ObjectId(),
permission: OwnerRoleVal
});
});
it('defaults to dry-run when the flag is omitted', async () => {
const rootUser = await getRootUser();
const response = await Call(cleanupDanglingResourcePermissionsHandler, {
auth: rootUser,
body: {
batchSize: 4,
maxScan: 100,
sampleLimit: 0
}
});
expect(response.error).toBeUndefined();
expect(response.data).toMatchObject({
dryRun: true,
scannedPermissionCount: 1,
danglingPermissionCount: 1,
deletedPermissionCount: 0,
samples: []
});
expect(await MongoResourcePermission.countDocuments()).toBe(1);
});
});
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment