Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit 980b4d3d by Finley Ge Committed by GitHub
Options

Dataset Permission (#1786) 

* feat: dataset controllers

feat: dataset schema

fix: add missing type to dataset schema
Signed-off-by: FinleyGe <m13203533462@163.com>

* feat: dataset list api

Signed-off-by: FinleyGe <m13203533462@163.com>

* chore: all dataset api

Signed-off-by: FinleyGe <m13203533462@163.com>

* feat: new auth dataset method

Signed-off-by: FinleyGe <m13203533462@163.com>

* chore: use new auth method in detail, paths.
feat: add new param defaultPermission to create api

Signed-off-by: FinleyGe <m13203533462@163.com>

* chore: app auth params

Signed-off-by: FinleyGe <m13203533462@163.com>

* chore: use new auth method

Signed-off-by: FinleyGe <m13203533462@163.com>

* feat: new auth collection and file method

Signed-off-by: FinleyGe <m13203533462@163.com>

* chore: dataset collection api new auth

Signed-off-by: FinleyGe <m13203533462@163.com>

* chore: create/*.ts auth

Signed-off-by: FinleyGe <m13203533462@163.com>

* chore: dataset auth

Signed-off-by: FinleyGe <m13203533462@163.com>

* fix: import paths

Signed-off-by: FinleyGe <m13203533462@163.com>

* feat: dataset collaborator

Signed-off-by: FinleyGe <m13203533462@163.com>

* chore: dataset frontend

feat: dataset list frontend

feat: dataset detail
Signed-off-by: FinleyGe <m13203533462@163.com>

* feat: finish the dataset permission

fix: ts errors
Signed-off-by: FinleyGe <m13203533462@163.com>

* fix: empty response of collection api

Signed-off-by: FinleyGe <m13203533462@163.com>

* chore: adjust the code

* chore: adjust the code

* chore: i18n

* fix: ts error

* fix: fe CollectionCard permission

---------

Signed-off-by: FinleyGe <m13203533462@163.com>
parent 2b25e3cc
Showing with 1413 additions and 1339 deletions
packages/global/common/error/code/common.ts
......@@ -4,7 +4,8 @@ import { ErrType } from '../errorCode';
const startCode = 507000;
export enum CommonErrEnum {
fileNotFound = 'fileNotFound',
unAuthFile = 'unAuthFile'
unAuthFile = 'unAuthFile',
missingParams = 'missingParams'
}
const datasetErr = [
{
......@@ -14,6 +15,10 @@ const datasetErr = [
{
statusText: CommonErrEnum.unAuthFile,
message: 'error.unAuthFile'
},
{
statusText: CommonErrEnum.missingParams,
message: 'error.missingParams'
}
];
export default datasetErr.reduce((acc, cur, index) => {
......
packages/global/common/error/code/dataset.ts
......@@ -2,6 +2,7 @@ import { ErrType } from '../errorCode';
/* dataset: 501000 */
export enum DatasetErrEnum {
unExist = 'unExistDataset',
unAuthDataset = 'unAuthDataset',
unCreateCollection = 'unCreateCollection',
unAuthDatasetCollection = 'unAuthDatasetCollection',
......@@ -12,6 +13,10 @@ export enum DatasetErrEnum {
}
const datasetErr = [
{
statusText: DatasetErrEnum.unExist,
message: 'core.dataset.error.unExistDataset'
},
{
statusText: DatasetErrEnum.unAuthDataset,
message: 'core.dataset.error.unAuthDataset'
},
......
packages/global/core/dataset/api.d.ts
......@@ -9,12 +9,13 @@ export type DatasetUpdateBody = {
name?: string;
avatar?: string;
intro?: string;
permission?: DatasetSchemaType['permission'];
permission?: DatasetSchemaType['permission']; // TODO: Should be deleted.
agentModel?: LLMModelItemType;
status?: DatasetSchemaType['status'];
websiteConfig?: DatasetSchemaType['websiteConfig'];
externalReadUrl?: DatasetSchemaType['externalReadUrl'];
defaultPermission?: DatasetSchemaType['defaultPermission'];
};
/* ================= collection ===================== */
......
packages/global/core/dataset/collaborator.d.ts 0 → 100644
import { UpdateClbPermissionProps } from '../../support/permission/collaborator';
import { PermissionValueType } from '../../support/permission/type';
export type UpdateDatasetCollaboratorBody = UpdateClbPermissionProps & {
datasetId: string;
};
export type DatasetCollaboratorDeleteParams = {
datasetId: string;
tmbId: string;
};
packages/global/core/dataset/type.d.ts
import { PermissionValueType } from 'support/permission/type';
import type { LLMModelItemType, VectorModelItemType } from '../../core/ai/model.d';
import { PermissionTypeEnum } from '../../support/permission/constant';
import { PushDatasetDataChunkProps } from './api';
......@@ -8,6 +9,8 @@ import {
SearchScoreTypeEnum,
TrainingModeEnum
} from './constants';
import { DatasetPermission } from '../../support/permission/dataset/controller';
import { Permission } from '../../support/permission/controller';
/* schema */
export type DatasetSchemaType = {
......@@ -24,7 +27,7 @@ export type DatasetSchemaType = {
intro: string;
type: DatasetTypeEnum;
status: `${DatasetStatusEnum}`;
permission: `${PermissionTypeEnum}`;
permission: DatasetPermission;
// metadata
websiteConfig?: {
......@@ -32,6 +35,7 @@ export type DatasetSchemaType = {
selector: string;
};
externalReadUrl?: string;
defaultPermission: PermissionValueType;
};
export type DatasetCollectionSchemaType = {
......@@ -132,24 +136,22 @@ export type DatasetListItemType = {
name: string;
intro: string;
type: DatasetTypeEnum;
isOwner: boolean;
canWrite: boolean;
permission: `${PermissionTypeEnum}`;
permission: DatasetPermission;
vectorModel: VectorModelItemType;
defaultPermission: PermissionValueType;
};
export type DatasetItemType = Omit<DatasetSchemaType, 'vectorModel' | 'agentModel'> & {
vectorModel: VectorModelItemType;
agentModel: LLMModelItemType;
isOwner: boolean;
canWrite: boolean;
};
/* ================= collection ===================== */
export type DatasetCollectionItemType = CollectionWithDatasetType & {
canWrite: boolean;
sourceName: string;
sourceId?: string;
file?: DatasetFileSchema;
permission: DatasetPermission;
};
/* ================= data ===================== */
......
packages/global/support/permission/dataset/constant.ts 0 → 100644
import { NullPermission, PermissionKeyEnum, PermissionList } from '../constant';
export enum DatasetPermissionKeyEnum {}
export const DatasetPermissionList = {
[PermissionKeyEnum.read]: {
...PermissionList[PermissionKeyEnum.read],
description: '可查看知识库内容'
},
[PermissionKeyEnum.write]: {
...PermissionList[PermissionKeyEnum.write],
description: '可增加和变更知识库内容'
},
[PermissionKeyEnum.manage]: {
...PermissionList[PermissionKeyEnum.manage],
description: '可管理整个知识库数据和信息'
}
};
export const DatasetDefaultPermission = NullPermission;
packages/global/support/permission/dataset/controller.ts 0 → 100644
import { NullPermission } from '../constant';
import { PerConstructPros, Permission } from '../controller';
export class DatasetPermission extends Permission {
constructor(props?: PerConstructPros) {
if (!props) {
props = {
per: NullPermission
};
} else if (!props?.per) {
props.per = NullPermission;
}
super(props);
}
}
packages/service/core/dataset/schema.ts
......@@ -12,6 +12,7 @@ import {
TeamMemberCollectionName
} from '@fastgpt/global/support/user/team/constant';
import { PermissionTypeEnum, PermissionTypeMap } from '@fastgpt/global/support/permission/constant';
import { DatasetDefaultPermission } from '@fastgpt/global/support/permission/dataset/constant';
export const DatasetCollectionName = 'datasets';
......@@ -90,7 +91,11 @@ const DatasetSchema = new Schema({
}
}
},
externalReadUrl: String
externalReadUrl: String,
defaultPermission: {
type: Number,
default: DatasetDefaultPermission
}
});
try {
......
packages/service/core/workflow/dispatch/plugin/run.ts
......@@ -39,7 +39,6 @@ export const dispatchRunPlugin = async (props: RunPluginProps): Promise<RunPlugi
if (source === PluginSourceEnum.personal) {
await authAppByTmbId({
appId: pluginId,
teamId: workflowApp.teamId,
tmbId: workflowApp.tmbId,
per: ReadPermissionVal
});
......
packages/service/core/workflow/dispatch/tools/runApp.ts
......@@ -2,7 +2,6 @@ import type { ChatItemType } from '@fastgpt/global/core/chat/type.d';
import type { ModuleDispatchProps } from '@fastgpt/global/core/workflow/type/index.d';
import { SelectAppItemType } from '@fastgpt/global/core/workflow/type/index.d';
import { dispatchWorkFlow } from '../index';
import { MongoApp } from '../../../../core/app/schema';
import { responseWrite } from '../../../../common/response';
import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants';
import { SseResponseEventEnum } from '@fastgpt/global/core/workflow/runtime/constants';
......@@ -48,7 +47,6 @@ export const dispatchAppRequest = async (props: Props): Promise<Response> => {
// 检查该工作流的tmb是否有调用该app的权限（不是校验对话的人，是否有权限）
const { app: appData } = await authAppByTmbId({
appId: app.id,
teamId: workflowApp.teamId,
tmbId: workflowApp.tmbId,
per: ReadPermissionVal
});
......
packages/service/support/permission/app/auth.ts
......@@ -12,17 +12,15 @@ import { AuthResponseType } from '../type/auth.d';
import { PermissionValueType } from '@fastgpt/global/support/permission/type';
export const authAppByTmbId = async ({
teamId,
tmbId,
appId,
per
}: {
teamId: string;
tmbId: string;
appId: string;
per: PermissionValueType;
}) => {
const { permission: tmbPer } = await getTmbInfoByTmbId({ tmbId });
const { teamId, permission: tmbPer } = await getTmbInfoByTmbId({ tmbId });
const app = await (async () => {
// get app and per
......@@ -68,10 +66,9 @@ export const authApp = async ({
}
> => {
const result = await parseHeaderCert(props);
const { teamId, tmbId } = result;
const { tmbId } = result;
const { app } = await authAppByTmbId({
teamId,
tmbId,
appId,
per
......
packages/service/support/permission/auth/dataset.ts → packages/service/support/permission/dataset/auth.ts
import { AuthModeType } from '../type';
import { parseHeaderCert } from '../controller';
import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
import { MongoDataset } from '../../../core/dataset/schema';
import { getCollectionWithDataset } from '../../../core/dataset/controller';
import { PermissionTypeEnum } from '@fastgpt/global/support/permission/constant';
import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant';
import { AuthResponseType } from '@fastgpt/global/support/permission/type';
import { PermissionValueType } from '@fastgpt/global/support/permission/type';
import { getResourcePermission, parseHeaderCert } from '../controller';
import { AuthPropsType, AuthResponseType } from '../type/auth';
import {
CollectionWithDatasetType,
DatasetDataItemType,
DatasetFileSchema,
DatasetSchemaType
} from '@fastgpt/global/core/dataset/type';
import { getTmbInfoByTmbId } from '../../user/team/controller';
import { MongoDataset } from '../../../core/dataset/schema';
import { PerResourceTypeEnum } from '@fastgpt/global/support/permission/constant';
import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
import { DatasetPermission } from '@fastgpt/global/support/permission/dataset/controller';
import { getCollectionWithDataset } from '../../../core/dataset/controller';
import { MongoDatasetCollection } from '../../../core/dataset/collection/schema';
import { getFileById } from '../../../common/file/gridfs/controller';
import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
import { getTmbInfoByTmbId } from '../../user/team/controller';
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
import { MongoDatasetCollection } from '../../../core/dataset/collection/schema';
import { MongoDatasetData } from '../../../core/dataset/data/schema';
export async function authDatasetByTmbId({
teamId,
tmbId,
datasetId,
per
}: {
teamId: string;
tmbId: string;
datasetId: string;
per: AuthModeType['per'];
per: PermissionValueType;
}) {
const { role } = await getTmbInfoByTmbId({ tmbId });
const { dataset, isOwner, canWrite } = await (async () => {
const dataset = await MongoDataset.findOne({ _id: datasetId, teamId }).lean();
const { teamId, permission: tmbPer } = await getTmbInfoByTmbId({ tmbId });
const dataset = await (async () => {
// get app and per
const [dataset, rp] = await Promise.all([
MongoDataset.findOne({ _id: datasetId, teamId }).lean(),
getResourcePermission({
teamId,
tmbId,
resourceId: datasetId,
resourceType: PerResourceTypeEnum.dataset
}) // this could be null
]);
if (!dataset) {
return Promise.reject(DatasetErrEnum.unAuthDataset);
return Promise.reject(DatasetErrEnum.unExist);
}
const isOwner =
role !== TeamMemberRoleEnum.visitor &&
(String(dataset.tmbId) === tmbId || role === TeamMemberRoleEnum.owner);
const canWrite =
isOwner ||
(role !== TeamMemberRoleEnum.visitor && dataset.permission === PermissionTypeEnum.public);
if (per === 'r') {
if (!isOwner && dataset.permission !== PermissionTypeEnum.public) {
return Promise.reject(DatasetErrEnum.unAuthDataset);
}
}
if (per === 'w' && !canWrite) {
return Promise.reject(DatasetErrEnum.unAuthDataset);
}
if (per === 'owner' && !isOwner) {
const isOwner = tmbPer.isOwner || String(dataset.tmbId) === tmbId;
const Per = new DatasetPermission({
per: rp?.permission ?? dataset.defaultPermission,
isOwner
});
if (!Per.checkPer(per)) {
return Promise.reject(DatasetErrEnum.unAuthDataset);
}
return { dataset, isOwner, canWrite };
return {
...dataset,
permission: Per
};
})();
return {
dataset,
isOwner,
canWrite
};
return { dataset: dataset };
}
// Auth Dataset
export async function authDataset({
datasetId,
per = 'owner',
per,
...props
}: AuthModeType & {
}: AuthPropsType & {
datasetId: string;
}): Promise<
AuthResponseType & {
AuthResponseType<DatasetPermission> & {
dataset: DatasetSchemaType;
}
> {
const result = await parseHeaderCert(props);
const { teamId, tmbId } = result;
const { dataset, isOwner, canWrite } = await authDatasetByTmbId({
teamId,
const { teamId, tmbId } = await parseHeaderCert(props);
const { dataset } = await authDatasetByTmbId({
tmbId,
datasetId,
per
});
return {
...result,
teamId,
tmbId,
dataset,
isOwner,
canWrite
permission: dataset.permission
};
}
/*
Read: in team and dataset permission is public
Write: in team, not visitor and dataset permission is public
*/
// the temporary solution for authDatasetCollection is getting the
export async function authDatasetCollection({
collectionId,
per = 'owner',
per,
...props
}: AuthModeType & {
}: AuthPropsType & {
collectionId: string;
}): Promise<
AuthResponseType & {
AuthResponseType<DatasetPermission> & {
collection: CollectionWithDatasetType;
}
> {
const { teamId, tmbId } = await parseHeaderCert(props);
const { role } = await getTmbInfoByTmbId({ tmbId });
const collection = await getCollectionWithDataset(collectionId);
const { collection, isOwner, canWrite } = await (async () => {
const collection = await getCollectionWithDataset(collectionId);
if (!collection || String(collection.teamId) !== teamId) {
return Promise.reject(DatasetErrEnum.unAuthDatasetCollection);
}
const isOwner = String(collection.tmbId) === tmbId || role === TeamMemberRoleEnum.owner;
const canWrite =
isOwner ||
(role !== TeamMemberRoleEnum.visitor &&
collection.datasetId.permission === PermissionTypeEnum.public);
if (per === 'r') {
if (!isOwner && collection.datasetId.permission !== PermissionTypeEnum.public) {
return Promise.reject(DatasetErrEnum.unAuthDatasetCollection);
}
}
if (per === 'w' && !canWrite) {
return Promise.reject(DatasetErrEnum.unAuthDatasetCollection);
}
if (per === 'owner' && !isOwner) {
return Promise.reject(DatasetErrEnum.unAuthDatasetCollection);
}
if (!collection) {
return Promise.reject(DatasetErrEnum.unExist);
}
return {
collection,
isOwner,
canWrite
};
})();
const { dataset } = await authDatasetByTmbId({
tmbId,
datasetId: collection.datasetId._id,
per
});
return {
teamId,
tmbId,
collection,
isOwner,
canWrite
permission: dataset.permission
};
}
export async function authDatasetFile({
fileId,
per = 'owner',
per,
...props
}: AuthModeType & {
}: AuthPropsType & {
fileId: string;
}): Promise<
AuthResponseType & {
AuthResponseType<DatasetPermission> & {
file: DatasetFileSchema;
}
> {
......@@ -180,9 +155,8 @@ export async function authDatasetFile({
return Promise.reject(DatasetErrEnum.unAuthDatasetFile);
}
// file role = collection role
try {
const { isOwner, canWrite } = await authDatasetCollection({
const { permission } = await authDatasetCollection({
...props,
collectionId: collection._id,
per
......@@ -192,10 +166,48 @@ export async function authDatasetFile({
teamId,
tmbId,
file,
isOwner,
canWrite
permission
};
} catch (error) {
return Promise.reject(DatasetErrEnum.unAuthDatasetFile);
}
}
export async function authDatasetData({
dataId,
...props
}: AuthPropsType & {
dataId: string;
}) {
// get mongo dataset.data
const datasetData = await MongoDatasetData.findById(dataId);
if (!datasetData) {
return Promise.reject('core.dataset.error.Data not found');
}
const result = await authDatasetCollection({
...props,
collectionId: datasetData.collectionId
});
const data: DatasetDataItemType = {
id: String(datasetData._id),
teamId: datasetData.teamId,
q: datasetData.q,
a: datasetData.a,
chunkIndex: datasetData.chunkIndex,
indexes: datasetData.indexes,
datasetId: String(datasetData.datasetId),
collectionId: String(datasetData.collectionId),
sourceName: result.collection.name || '',
sourceId: result.collection?.fileId || result.collection?.rawLink,
isOwner: String(datasetData.tmbId) === result.tmbId,
canWrite: result.permission.hasWritePer
};
return {
...result,
datasetData: data
};
}
packages/service/support/permission/publish/authLink.ts
......@@ -31,7 +31,6 @@ export async function authOutLinkCrud({
}
const { app } = await authAppByTmbId({
teamId,
tmbId,
appId: outLink.appId,
per: ManagePermissionVal
......
packages/service/support/permission/type/auth.d.ts
......@@ -11,11 +11,11 @@ export type AuthPropsType = {
per: PermissionValueType;
};
export type AuthResponseType = {
export type AuthResponseType<T = Permission> = {
teamId: string;
tmbId: string;
authType?: `${AuthUserTypeEnum}`;
appId?: string;
apikey?: string;
permission: Permission;
permission: T;
};
pnpm-lock.yaml
This source diff could not be displayed because it is too large. You can view the blob instead.
projects/app/next.config.js
/** @type {import('next').NextConfig} */
const { i18n } = require('./next-i18next.config');
const path = require('path');
const isDev = process.env.NODE_ENV === 'development';
/** @type {import('next').NextConfig} */
const nextConfig = {
i18n,
output: 'standalone',
......
projects/app/src/global/core/dataset/api.d.ts
......@@ -11,6 +11,7 @@ import {
SearchDataResponseItemType
} from '@fastgpt/global/core/dataset/type';
import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import { PermissionValueType } from '@fastgpt/global/support/permission/type';
/* ================= dataset ===================== */
export type CreateDatasetParams = {
......@@ -21,6 +22,7 @@ export type CreateDatasetParams = {
avatar: string;
vectorModel?: string;
agentModel?: string;
defaultPermission?: PermissionValueType;
};
export type RebuildEmbeddingProps = {
......
projects/app/src/global/core/dataset/type.d.ts
......@@ -3,6 +3,7 @@ import {
DatasetCollectionSchemaType,
DatasetDataSchemaType
} from '@fastgpt/global/core/dataset/type.d';
import { DatasetPermission } from '@fastgpt/global/support/permission/dataset/controller';
/* ================= dataset ===================== */
......@@ -18,7 +19,7 @@ export type DatasetCollectionsListItemType = {
trainingAmount: number;
fileId?: string;
rawLink?: string;
canWrite: boolean;
permission: DatasetPermission;
};
/* ================= data ===================== */
......
projects/app/src/pages/api/core/dataset/allDataset.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import type { NextApiRequest } from 'next';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { getVectorModel } from '@fastgpt/service/core/ai/model';
import type { DatasetSimpleItemType } from '@fastgpt/global/core/dataset/type.d';
import { mongoRPermission } from '@fastgpt/global/support/permission/utils';
import { authUserPer } from '@fastgpt/service/support/permission/user/auth';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { NextAPI } from '@/service/middleware/entry';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import {
PerResourceTypeEnum,
ReadPermissionVal
} from '@fastgpt/global/support/permission/constant';
import { MongoResourcePermission } from '@fastgpt/service/support/permission/schema';
import { DatasetPermission } from '@fastgpt/global/support/permission/dataset/controller';
import { authUserPer } from '@fastgpt/service/support/permission/user/auth';
/* get all dataset by teamId or tmbId */
async function handler(
req: NextApiRequest,
res: NextApiResponse<any>
): Promise<DatasetSimpleItemType[]> {
// 凭证校验
const { teamId, tmbId, permission } = await authUserPer({
async function handler(req: NextApiRequest): Promise<DatasetSimpleItemType[]> {
const {
teamId,
tmbId,
permission: tmbPer
} = await authUserPer({
req,
authToken: true,
authApiKey: true,
per: ReadPermissionVal
});
const datasets = await MongoDataset.find({
...mongoRPermission({ teamId, tmbId, permission }),
type: { $ne: DatasetTypeEnum.folder }
}).lean();
const [myDatasets, rpList] = await Promise.all([
MongoDataset.find({
teamId,
type: {
$ne: DatasetTypeEnum.folder
}
})
.sort({
updateTime: -1
})
.lean(),
MongoResourcePermission.find({
resourceType: PerResourceTypeEnum.dataset,
teamId,
tmbId
}).lean()
]);
const filterDatasets = myDatasets
.map((dataset) => {
const perVal = rpList.find(
(item) => String(item.resourceId) === String(dataset._id)
)?.permission;
const Per = new DatasetPermission({
per: perVal ?? dataset.defaultPermission,
isOwner: String(dataset.tmbId) === tmbId || tmbPer.isOwner
});
return {
...dataset,
permission: Per
};
})
.filter((app) => app.permission.hasReadPer);
return datasets.map((item) => ({
return filterDatasets.map((item) => ({
_id: item._id,
avatar: item.avatar,
name: item.name,
......
projects/app/src/pages/api/core/dataset/collection/create.ts
/*
Create one dataset collection
*/
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import type { NextApiRequest } from 'next';
import type { CreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api.d';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
import { createOneCollection } from '@fastgpt/service/core/dataset/collection/controller';
import { NextAPI } from '@/service/middleware/entry';
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const body = req.body as CreateDatasetCollectionParams;
async function handler(req: NextApiRequest) {
const body = req.body as CreateDatasetCollectionParams;
const { teamId, tmbId } = await authDataset({
req,
authToken: true,
authApiKey: true,
datasetId: body.datasetId,
per: 'w'
});
const { teamId, tmbId } = await authDataset({
req,
authToken: true,
authApiKey: true,
datasetId: body.datasetId,
per: WritePermissionVal
});
const { _id } = await createOneCollection({
...body,
teamId,
tmbId
});
jsonRes(res, {
data: _id
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
const { _id } = await createOneCollection({
...body,
teamId,
tmbId
});
return _id;
}
export default NextAPI(handler);
projects/app/src/pages/api/core/dataset/collection/create/csvTable.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import type { NextApiRequest } from 'next';
import { readFileContentFromMongo } from '@fastgpt/service/common/file/gridfs/controller';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
import { FileIdCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api';
import { createOneCollection } from '@fastgpt/service/core/dataset/collection/controller';
import {
......@@ -18,97 +16,88 @@ import { createTrainingUsage } from '@fastgpt/service/support/wallet/usage/contr
import { UsageSourceEnum } from '@fastgpt/global/support/wallet/usage/constants';
import { getLLMModel, getVectorModel } from '@fastgpt/service/core/ai/model';
import { rawText2Chunks } from '@fastgpt/service/core/dataset/read';
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
import { NextAPI } from '@/service/middleware/entry';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
async function handler(req: NextApiRequest) {
const { datasetId, parentId, fileId } = req.body as FileIdCreateDatasetCollectionParams;
const trainingType = TrainingModeEnum.chunk;
const { teamId, tmbId, dataset } = await authDataset({
req,
authToken: true,
authApiKey: true,
per: WritePermissionVal,
datasetId: datasetId
});
try {
await connectToDatabase();
// 1. read file
const { rawText, filename } = await readFileContentFromMongo({
teamId,
bucketName: BucketNameEnum.dataset,
fileId,
isQAImport: true
});
console.log(rawText);
// 2. split chunks
const chunks = rawText2Chunks({
rawText,
isQAImport: true
});
const { teamId, tmbId, dataset } = await authDataset({
req,
authToken: true,
authApiKey: true,
per: 'w',
datasetId: datasetId
});
// 3. auth limit
await checkDatasetLimit({
teamId,
insertLen: predictDataLimitLength(trainingType, chunks)
});
// 1. read file
const { rawText, filename } = await readFileContentFromMongo({
await mongoSessionRun(async (session) => {
// 4. create collection
const { _id: collectionId } = await createOneCollection({
teamId,
bucketName: BucketNameEnum.dataset,
tmbId,
name: filename,
parentId,
datasetId,
type: DatasetCollectionTypeEnum.file,
fileId,
isQAImport: true
});
console.log(rawText);
// 2. split chunks
const chunks = rawText2Chunks({
rawText,
isQAImport: true
// special metadata
trainingType,
chunkSize: 0,
session
});
// 3. auth limit
await checkDatasetLimit({
// 5. create training bill
const { billId } = await createTrainingUsage({
teamId,
insertLen: predictDataLimitLength(trainingType, chunks)
tmbId,
appName: filename,
billSource: UsageSourceEnum.training,
vectorModel: getVectorModel(dataset.vectorModel)?.name,
agentModel: getLLMModel(dataset.agentModel)?.name,
session
});
await mongoSessionRun(async (session) => {
// 4. create collection
const { _id: collectionId } = await createOneCollection({
teamId,
tmbId,
name: filename,
parentId,
datasetId,
type: DatasetCollectionTypeEnum.file,
fileId,
// special metadata
trainingType,
chunkSize: 0,
session
});
// 5. create training bill
const { billId } = await createTrainingUsage({
teamId,
tmbId,
appName: filename,
billSource: UsageSourceEnum.training,
vectorModel: getVectorModel(dataset.vectorModel)?.name,
agentModel: getLLMModel(dataset.agentModel)?.name,
session
});
// 6. insert to training queue
await pushDataListToTrainingQueue({
teamId,
tmbId,
datasetId: dataset._id,
collectionId,
agentModel: dataset.agentModel,
vectorModel: dataset.vectorModel,
trainingMode: trainingType,
billId,
data: chunks.map((chunk, index) => ({
q: chunk.q,
a: chunk.a,
chunkIndex: index
})),
session
});
return collectionId;
// 6. insert to training queue
await pushDataListToTrainingQueue({
teamId,
tmbId,
datasetId: dataset._id,
collectionId,
agentModel: dataset.agentModel,
vectorModel: dataset.vectorModel,
trainingMode: trainingType,
billId,
data: chunks.map((chunk, index) => ({
q: chunk.q,
a: chunk.a,
chunkIndex: index
})),
session
});
jsonRes(res);
} catch (error) {
jsonRes(res, {
code: 500,
error
});
}
return collectionId;
});
}
export default NextAPI(handler);
projects/app/src/pages/api/core/dataset/collection/create/fileId.ts
import type { NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { readFileContentFromMongo } from '@fastgpt/service/common/file/gridfs/controller';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
import { FileIdCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api';
import { createOneCollection } from '@fastgpt/service/core/dataset/collection/controller';
import {
......@@ -23,11 +20,9 @@ import { MongoRawTextBuffer } from '@fastgpt/service/common/buffer/rawText/schem
import { rawText2Chunks } from '@fastgpt/service/core/dataset/read';
import { NextAPI } from '@/service/middleware/entry';
import { ApiRequestProps } from '@fastgpt/service/type/next';
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
async function handler(
req: ApiRequestProps<FileIdCreateDatasetCollectionParams>,
res: NextApiResponse<any>
) {
async function handler(req: ApiRequestProps<FileIdCreateDatasetCollectionParams>) {
const {
fileId,
trainingType = TrainingModeEnum.chunk,
......@@ -37,13 +32,11 @@ async function handler(
...body
} = req.body;
await connectToDatabase();
const { teamId, tmbId, dataset } = await authDataset({
req,
authToken: true,
authApiKey: true,
per: 'w',
per: WritePermissionVal,
datasetId: body.datasetId
});
......@@ -137,13 +130,10 @@ async function handler(
}
);
// remove buffer
await MongoRawTextBuffer.deleteOne({ sourceId: fileId });
return collectionId;
});
// remove buffer
await MongoRawTextBuffer.deleteOne({ sourceId: fileId });
jsonRes(res);
}
export default NextAPI(handler);
projects/app/src/pages/api/core/dataset/collection/create/link.ts
/*
Create one dataset collection
*/
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import type { NextApiRequest } from 'next';
import type { LinkCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api.d';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
import { createOneCollection } from '@fastgpt/service/core/dataset/collection/controller';
import {
TrainingModeEnum,
......@@ -18,83 +13,75 @@ import { UsageSourceEnum } from '@fastgpt/global/support/wallet/usage/constants'
import { getLLMModel, getVectorModel } from '@fastgpt/service/core/ai/model';
import { reloadCollectionChunks } from '@fastgpt/service/core/dataset/collection/utils';
import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
import { NextAPI } from '@/service/middleware/entry';
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const {
link,
trainingType = TrainingModeEnum.chunk,
chunkSize = 512,
async function handler(req: NextApiRequest) {
const {
link,
trainingType = TrainingModeEnum.chunk,
chunkSize = 512,
chunkSplitter,
qaPrompt,
...body
} = req.body as LinkCreateDatasetCollectionParams;
const { teamId, tmbId, dataset } = await authDataset({
req,
authToken: true,
authApiKey: true,
datasetId: body.datasetId,
per: WritePermissionVal
});
// 1. check dataset limit
await checkDatasetLimit({
teamId,
insertLen: predictDataLimitLength(trainingType, new Array(10))
});
await mongoSessionRun(async (session) => {
// 2. create collection
const collection = await createOneCollection({
...body,
name: link,
teamId,
tmbId,
type: DatasetCollectionTypeEnum.link,
trainingType,
chunkSize,
chunkSplitter,
qaPrompt,
...body
} = req.body as LinkCreateDatasetCollectionParams;
const { teamId, tmbId, dataset } = await authDataset({
req,
authToken: true,
authApiKey: true,
datasetId: body.datasetId,
per: 'w'
rawLink: link,
session
});
// 1. check dataset limit
await checkDatasetLimit({
// 3. create bill and start sync
const { billId } = await createTrainingUsage({
teamId,
insertLen: predictDataLimitLength(trainingType, new Array(10))
tmbId,
appName: 'core.dataset.collection.Sync Collection',
billSource: UsageSourceEnum.training,
vectorModel: getVectorModel(dataset.vectorModel).name,
agentModel: getLLMModel(dataset.agentModel).name,
session
});
const { _id: collectionId } = await mongoSessionRun(async (session) => {
// 2. create collection
const collection = await createOneCollection({
...body,
name: link,
teamId,
tmbId,
type: DatasetCollectionTypeEnum.link,
trainingType,
chunkSize,
chunkSplitter,
qaPrompt,
rawLink: link,
session
});
// 3. create bill and start sync
const { billId } = await createTrainingUsage({
teamId,
tmbId,
appName: 'core.dataset.collection.Sync Collection',
billSource: UsageSourceEnum.training,
vectorModel: getVectorModel(dataset.vectorModel).name,
agentModel: getLLMModel(dataset.agentModel).name,
session
});
// load
await reloadCollectionChunks({
collection: {
...collection.toObject(),
datasetId: dataset
},
tmbId,
billId,
session
});
return collection;
// load
await reloadCollectionChunks({
collection: {
...collection.toObject(),
datasetId: dataset
},
tmbId,
billId,
session
});
jsonRes(res, {
data: { collectionId }
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
return collection;
});
}
export default NextAPI(handler);
projects/app/src/pages/api/core/dataset/collection/create/localFile.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { uploadFile } from '@fastgpt/service/common/file/gridfs/controller';
import { getUploadModel } from '@fastgpt/service/common/file/multer';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
import { FileCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api';
import { removeFilesByPaths } from '@fastgpt/service/common/file/utils';
import { createOneCollection } from '@fastgpt/service/core/dataset/collection/controller';
......@@ -23,6 +22,7 @@ import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
import { MongoImage } from '@fastgpt/service/common/file/image/schema';
import { readRawTextByLocalFile } from '@fastgpt/service/common/file/read/utils';
import { NextAPI } from '@/service/middleware/entry';
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
/**
......@@ -49,7 +49,7 @@ async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
req,
authToken: true,
authApiKey: true,
per: 'w',
per: WritePermissionVal,
datasetId: data.datasetId
});
......@@ -168,9 +168,7 @@ async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
};
});
jsonRes(res, {
data: { collectionId, results: insertResults }
});
return { collectionId, results: insertResults };
} catch (error) {
removeFilesByPaths(filePaths);
......
projects/app/src/pages/api/core/dataset/collection/create/text.ts
/*
Create one dataset collection
*/
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import type { NextApiRequest } from 'next';
import type { TextCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api.d';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
import { createOneCollection } from '@fastgpt/service/core/dataset/collection/controller';
import {
TrainingModeEnum,
......@@ -20,102 +15,94 @@ import { createTrainingUsage } from '@fastgpt/service/support/wallet/usage/contr
import { UsageSourceEnum } from '@fastgpt/global/support/wallet/usage/constants';
import { getLLMModel, getVectorModel } from '@fastgpt/service/core/ai/model';
import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
import { NextAPI } from '@/service/middleware/entry';
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
async function handler(req: NextApiRequest) {
const {
name,
text,
trainingType = TrainingModeEnum.chunk,
chunkSize = 512,
chunkSplitter,
qaPrompt,
...body
} = req.body as TextCreateDatasetCollectionParams;
const { teamId, tmbId, dataset } = await authDataset({
req,
authToken: true,
authApiKey: true,
datasetId: body.datasetId,
per: WritePermissionVal
});
// 1. split text to chunks
const { chunks } = splitText2Chunks({
text,
chunkLen: chunkSize,
overlapRatio: trainingType === TrainingModeEnum.chunk ? 0.2 : 0,
customReg: chunkSplitter ? [chunkSplitter] : []
});
// 2. check dataset limit
await checkDatasetLimit({
teamId,
insertLen: predictDataLimitLength(trainingType, chunks)
});
const createResult = await mongoSessionRun(async (session) => {
// 3. create collection
const { _id: collectionId } = await createOneCollection({
...body,
teamId,
tmbId,
type: DatasetCollectionTypeEnum.virtual,
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const {
name,
text,
trainingType = TrainingModeEnum.chunk,
chunkSize = 512,
trainingType,
chunkSize,
chunkSplitter,
qaPrompt,
...body
} = req.body as TextCreateDatasetCollectionParams;
const { teamId, tmbId, dataset } = await authDataset({
req,
authToken: true,
authApiKey: true,
datasetId: body.datasetId,
per: 'w'
hashRawText: hashStr(text),
rawTextLength: text.length,
session
});
// 1. split text to chunks
const { chunks } = splitText2Chunks({
text,
chunkLen: chunkSize,
overlapRatio: trainingType === TrainingModeEnum.chunk ? 0.2 : 0,
customReg: chunkSplitter ? [chunkSplitter] : []
// 4. create training bill
const { billId } = await createTrainingUsage({
teamId,
tmbId,
appName: name,
billSource: UsageSourceEnum.training,
vectorModel: getVectorModel(dataset.vectorModel)?.name,
agentModel: getLLMModel(dataset.agentModel)?.name,
session
});
// 2. check dataset limit
await checkDatasetLimit({
// 5. push chunks to training queue
const insertResults = await pushDataListToTrainingQueue({
teamId,
insertLen: predictDataLimitLength(trainingType, chunks)
tmbId,
datasetId: dataset._id,
collectionId,
agentModel: dataset.agentModel,
vectorModel: dataset.vectorModel,
trainingMode: trainingType,
prompt: qaPrompt,
billId,
data: chunks.map((text, index) => ({
q: text,
chunkIndex: index
})),
session
});
const createResult = await mongoSessionRun(async (session) => {
// 3. create collection
const { _id: collectionId } = await createOneCollection({
...body,
teamId,
tmbId,
type: DatasetCollectionTypeEnum.virtual,
name,
trainingType,
chunkSize,
chunkSplitter,
qaPrompt,
hashRawText: hashStr(text),
rawTextLength: text.length,
session
});
return { collectionId, results: insertResults };
});
// 4. create training bill
const { billId } = await createTrainingUsage({
teamId,
tmbId,
appName: name,
billSource: UsageSourceEnum.training,
vectorModel: getVectorModel(dataset.vectorModel)?.name,
agentModel: getLLMModel(dataset.agentModel)?.name,
session
});
// 5. push chunks to training queue
const insertResults = await pushDataListToTrainingQueue({
teamId,
tmbId,
datasetId: dataset._id,
collectionId,
agentModel: dataset.agentModel,
vectorModel: dataset.vectorModel,
trainingMode: trainingType,
prompt: qaPrompt,
billId,
data: chunks.map((text, index) => ({
q: text,
chunkIndex: index
})),
session
});
return { collectionId, results: insertResults };
});
jsonRes(res, {
data: createResult
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
return createResult;
}
export const config = {
......@@ -125,3 +112,5 @@ export const config = {
}
}
};
export default NextAPI(handler);
projects/app/src/pages/api/core/dataset/collection/delete.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import type { NextApiRequest } from 'next';
import { findCollectionAndChild } from '@fastgpt/service/core/dataset/collection/utils';
import { delCollectionAndRelatedSources } from '@fastgpt/service/core/dataset/collection/controller';
import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset';
import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
import { NextAPI } from '@/service/middleware/entry';
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
async function handler(req: NextApiRequest) {
const { id: collectionId } = req.query as { id: string };
const { id: collectionId } = req.query as { id: string };
if (!collectionId) {
throw new Error('CollectionIdId is required');
}
const { teamId, collection } = await authDatasetCollection({
req,
authToken: true,
authApiKey: true,
collectionId,
per: 'w'
});
if (!collectionId) {
return Promise.reject(CommonErrEnum.missingParams);
}
// find all delete id
const collections = await findCollectionAndChild({
teamId,
datasetId: collection.datasetId._id,
collectionId,
fields: '_id teamId datasetId fileId metadata'
});
const { teamId, collection } = await authDatasetCollection({
req,
authToken: true,
authApiKey: true,
collectionId,
per: WritePermissionVal
});
// delete
await mongoSessionRun((session) =>
delCollectionAndRelatedSources({
collections,
session
})
);
// find all delete id
const collections = await findCollectionAndChild({
teamId,
datasetId: collection.datasetId._id,
collectionId,
fields: '_id teamId datasetId fileId metadata'
});
jsonRes(res);
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
// delete
await mongoSessionRun((session) =>
delCollectionAndRelatedSources({
collections,
session
})
);
}
export default NextAPI(handler);
projects/app/src/pages/api/core/dataset/collection/detail.ts
/*
Get one dataset collection detail
*/
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset';
import { DatasetCollectionItemType } from '@fastgpt/global/core/dataset/type';
import type { NextApiRequest } from 'next';
import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
import { getFileById } from '@fastgpt/service/common/file/gridfs/controller';
import { getCollectionSourceData } from '@fastgpt/global/core/dataset/collection/utils';
import { NextAPI } from '@/service/middleware/entry';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { DatasetCollectionItemType } from '@fastgpt/global/core/dataset/type';
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { id } = req.query as { id: string };
async function handler(req: NextApiRequest): Promise<DatasetCollectionItemType> {
const { id } = req.query as { id: string };
if (!id) {
throw new Error('Id is required');
}
if (!id) {
return Promise.reject(CommonErrEnum.missingParams);
}
// 凭证校验
const { collection, canWrite } = await authDatasetCollection({
req,
authToken: true,
authApiKey: true,
collectionId: id,
per: 'r'
});
// 凭证校验
const { collection, permission } = await authDatasetCollection({
req,
authToken: true,
authApiKey: true,
collectionId: id,
per: ReadPermissionVal
});
// get file
const file = collection?.fileId
? await getFileById({ bucketName: BucketNameEnum.dataset, fileId: collection.fileId })
: undefined;
// get file
const file = collection?.fileId
? await getFileById({ bucketName: BucketNameEnum.dataset, fileId: collection.fileId })
: undefined;
jsonRes<DatasetCollectionItemType>(res, {
data: {
...collection,
canWrite,
...getCollectionSourceData(collection),
file
}
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
return {
...collection,
...getCollectionSourceData(collection),
permission,
file
};
}
export default NextAPI(handler);
projects/app/src/pages/api/core/dataset/collection/list.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import type { NextApiRequest } from 'next';
import { DatasetTrainingCollectionName } from '@fastgpt/service/core/dataset/training/schema';
import { Types } from '@fastgpt/service/common/mongo';
import type { DatasetCollectionsListItemType } from '@/global/core/dataset/type.d';
import type { GetDatasetCollectionsProps } from '@/global/core/api/datasetReq';
import { PagingData } from '@/types';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
import { DatasetDataCollectionName } from '@fastgpt/service/core/dataset/data/schema';
import { startTrainingQueue } from '@/service/core/dataset/training/utils';
import { NextAPI } from '@/service/middleware/entry';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
async function handler(req: NextApiRequest) {
let {
pageNum = 1,
pageSize = 10,
datasetId,
parentId = null,
searchText = '',
selectFolder = false,
simple = false
} = req.body as GetDatasetCollectionsProps;
searchText = searchText?.replace(/'/g, '');
pageSize = Math.min(pageSize, 30);
let {
pageNum = 1,
pageSize = 10,
datasetId,
parentId = null,
searchText = '',
selectFolder = false,
simple = false
} = req.body as GetDatasetCollectionsProps;
searchText = searchText?.replace(/'/g, '');
pageSize = Math.min(pageSize, 30);
// auth dataset and get my role
const { teamId, permission } = await authDataset({
req,
authToken: true,
authApiKey: true,
datasetId,
per: ReadPermissionVal
});
// auth dataset and get my role
const { teamId, tmbId, canWrite } = await authDataset({
req,
authToken: true,
authApiKey: true,
datasetId,
per: 'r'
});
const match = {
teamId: new Types.ObjectId(teamId),
datasetId: new Types.ObjectId(datasetId),
parentId: parentId ? new Types.ObjectId(parentId) : null,
...(selectFolder ? { type: DatasetCollectionTypeEnum.folder } : {}),
...(searchText
? {
name: new RegExp(searchText, 'i')
}
: {})
};
const match = {
teamId: new Types.ObjectId(teamId),
datasetId: new Types.ObjectId(datasetId),
parentId: parentId ? new Types.ObjectId(parentId) : null,
...(selectFolder ? { type: DatasetCollectionTypeEnum.folder } : {}),
...(searchText
? {
name: new RegExp(searchText, 'i')
}
: {})
// not count data amount
if (simple) {
const collections = await MongoDatasetCollection.find(match, '_id parentId type name')
.sort({
updateTime: -1
})
.lean();
return {
pageNum,
pageSize,
data: await Promise.all(
collections.map(async (item) => ({
...item,
dataAmount: 0,
trainingAmount: 0,
permission
}))
),
total: await MongoDatasetCollection.countDocuments(match)
};
}
// not count data amount
if (simple) {
const collections = await MongoDatasetCollection.find(match, '_id parentId type name')
.sort({
updateTime: -1
})
.lean();
return jsonRes<PagingData<DatasetCollectionsListItemType>>(res, {
data: {
pageNum,
pageSize,
data: await Promise.all(
collections.map(async (item) => ({
...item,
dataAmount: 0,
trainingAmount: 0,
canWrite // admin or team owner can write
}))
),
total: await MongoDatasetCollection.countDocuments(match)
}
});
}
const [collections, total]: [DatasetCollectionsListItemType[], number] = await Promise.all([
MongoDatasetCollection.aggregate([
{
$match: match
},
{
$sort: { updateTime: -1 }
},
{
$skip: (pageNum - 1) * pageSize
},
{
$limit: pageSize
},
// count training data
{
$lookup: {
from: DatasetTrainingCollectionName,
let: { id: '$_id', team_id: match.teamId, dataset_id: match.datasetId },
pipeline: [
{
$match: {
$expr: {
$and: [{ $eq: ['$teamId', '$$team_id'] }, { $eq: ['$collectionId', '$$id'] }]
}
const [collections, total]: [DatasetCollectionsListItemType[], number] = await Promise.all([
MongoDatasetCollection.aggregate([
{
$match: match
},
{
$sort: { updateTime: -1 }
},
{
$skip: (pageNum - 1) * pageSize
},
{
$limit: pageSize
},
// count training data
{
$lookup: {
from: DatasetTrainingCollectionName,
let: { id: '$_id', team_id: match.teamId, dataset_id: match.datasetId },
pipeline: [
{
$match: {
$expr: {
$and: [{ $eq: ['$teamId', '$$team_id'] }, { $eq: ['$collectionId', '$$id'] }]
}
},
{ $count: 'count' }
],
as: 'trainingCount'
}
},
// count collection total data
{
$lookup: {
from: DatasetDataCollectionName,
let: { id: '$_id', team_id: match.teamId, dataset_id: match.datasetId },
pipeline: [
{
$match: {
$expr: {
$and: [
{ $eq: ['$teamId', '$$team_id'] },
{ $eq: ['$datasetId', '$$dataset_id'] },
{ $eq: ['$collectionId', '$$id'] }
]
}
}
},
{ $count: 'count' }
],
as: 'trainingCount'
}
},
// count collection total data
{
$lookup: {
from: DatasetDataCollectionName,
let: { id: '$_id', team_id: match.teamId, dataset_id: match.datasetId },
pipeline: [
{
$match: {
$expr: {
$and: [
{ $eq: ['$teamId', '$$team_id'] },
{ $eq: ['$datasetId', '$$dataset_id'] },
{ $eq: ['$collectionId', '$$id'] }
]
}
},
{ $count: 'count' }
],
as: 'dataCount'
}
},
{
$project: {
_id: 1,
parentId: 1,
tmbId: 1,
name: 1,
type: 1,
status: 1,
updateTime: 1,
fileId: 1,
rawLink: 1,
dataAmount: {
$ifNull: [{ $arrayElemAt: ['$dataCount.count', 0] }, 0]
}
},
trainingAmount: {
$ifNull: [{ $arrayElemAt: ['$trainingCount.count', 0] }, 0]
}
{ $count: 'count' }
],
as: 'dataCount'
}
},
{
$project: {
_id: 1,
parentId: 1,
tmbId: 1,
name: 1,
type: 1,
status: 1,
updateTime: 1,
fileId: 1,
rawLink: 1,
dataAmount: {
$ifNull: [{ $arrayElemAt: ['$dataCount.count', 0] }, 0]
},
trainingAmount: {
$ifNull: [{ $arrayElemAt: ['$trainingCount.count', 0] }, 0]
}
}
]),
MongoDatasetCollection.countDocuments(match)
]);
const data = await Promise.all(
collections.map(async (item, i) => ({
...item,
canWrite: String(item.tmbId) === tmbId || canWrite
}))
);
}
]),
MongoDatasetCollection.countDocuments(match)
]);
if (data.find((item) => item.trainingAmount > 0)) {
startTrainingQueue();
}
const data = await Promise.all(
collections.map(async (item) => ({
...item,
permission
}))
);
// count collections
jsonRes<PagingData<DatasetCollectionsListItemType>>(res, {
data: {
pageNum,
pageSize,
data,
total
}
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
if (data.find((item) => item.trainingAmount > 0)) {
startTrainingQueue();
}
// count collections
return {
pageNum,
pageSize,
data,
total
};
}
export default NextAPI(handler);
projects/app/src/pages/api/core/dataset/collection/paths.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import type { ParentTreePathItemType } from '@fastgpt/global/common/parentFolder/type.d';
import type { NextApiRequest } from 'next';
import { getDatasetCollectionPaths } from '@fastgpt/service/core/dataset/collection/utils';
import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset';
import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
export default async function handler(req: NextApiRequest) {
const { parentId } = req.query as { parentId: string };
const { parentId } = req.query as { parentId: string };
if (!parentId) {
return jsonRes(res, {
data: []
});
}
if (!parentId) {
return [];
}
await authDatasetCollection({ req, authToken: true, collectionId: parentId, per: 'r' });
const paths = await getDatasetCollectionPaths({
parentId
});
await authDatasetCollection({
req,
authToken: true,
collectionId: parentId,
per: ReadPermissionVal
});
const paths = await getDatasetCollectionPaths({
parentId
});
jsonRes<ParentTreePathItemType[]>(res, {
data: paths
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
return paths;
}
projects/app/src/pages/api/core/dataset/collection/read.ts
import type { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next';
import type { ApiRequestProps } from '@fastgpt/service/type/next';
import { NextAPI } from '@/service/middleware/entry';
import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset';
import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { createFileToken } from '@fastgpt/service/support/permission/controller';
import { BucketNameEnum, ReadFileBaseUrl } from '@fastgpt/global/common/file/constants';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
export type readCollectionSourceQuery = {
collectionId: string;
......@@ -17,15 +18,14 @@ export type readCollectionSourceResponse = {
};
async function handler(
req: ApiRequestProps<readCollectionSourceBody, readCollectionSourceQuery>,
res: ApiResponseType<any>
req: ApiRequestProps<readCollectionSourceBody, readCollectionSourceQuery>
): Promise<readCollectionSourceResponse> {
const { collection, teamId, tmbId } = await authDatasetCollection({
req,
authToken: true,
authApiKey: true,
collectionId: req.query.collectionId,
per: 'r'
per: ReadPermissionVal
});
const sourceUrl = await (async () => {
......
projects/app/src/pages/api/core/dataset/collection/sync/link.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset';
import type { NextApiRequest } from 'next';
import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
import {
getCollectionAndRawText,
reloadCollectionChunks
......@@ -17,98 +15,90 @@ import { UsageSourceEnum } from '@fastgpt/global/support/wallet/usage/constants'
import { getLLMModel, getVectorModel } from '@fastgpt/service/core/ai/model';
import { createOneCollection } from '@fastgpt/service/core/dataset/collection/controller';
import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
import { NextAPI } from '@/service/middleware/entry';
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
async function handler(req: NextApiRequest) {
const { collectionId } = req.body as { collectionId: string };
const { collectionId } = req.body as { collectionId: string };
if (!collectionId) {
throw new Error('CollectionIdId is required');
}
const { collection, tmbId } = await authDatasetCollection({
req,
authToken: true,
collectionId,
per: 'w'
});
if (collection.type !== DatasetCollectionTypeEnum.link || !collection.rawLink) {
return Promise.reject(DatasetErrEnum.unLinkCollection);
}
if (!collectionId) {
return Promise.reject(CommonErrEnum.missingParams);
}
const { title, rawText, isSameRawText } = await getCollectionAndRawText({
collection
});
const { collection, tmbId } = await authDatasetCollection({
req,
authToken: true,
collectionId,
per: WritePermissionVal
});
if (isSameRawText) {
return jsonRes(res, {
data: DatasetCollectionSyncResultEnum.sameRaw
});
}
if (collection.type !== DatasetCollectionTypeEnum.link || !collection.rawLink) {
return Promise.reject(DatasetErrEnum.unLinkCollection);
}
/* Not the same original text, create and reload */
const { title, rawText, isSameRawText } = await getCollectionAndRawText({
collection
});
const vectorModelData = getVectorModel(collection.datasetId.vectorModel);
const agentModelData = getLLMModel(collection.datasetId.agentModel);
if (isSameRawText) {
return DatasetCollectionSyncResultEnum.sameRaw;
}
await mongoSessionRun(async (session) => {
// create training bill
const { billId } = await createTrainingUsage({
teamId: collection.teamId,
tmbId,
appName: 'core.dataset.collection.Sync Collection',
billSource: UsageSourceEnum.training,
vectorModel: vectorModelData.name,
agentModel: agentModelData.name,
session
});
/* Not the same original text, create and reload */
// create a collection and delete old
const newCol = await createOneCollection({
teamId: collection.teamId,
tmbId: collection.tmbId,
parentId: collection.parentId,
datasetId: collection.datasetId._id,
name: title || collection.name,
type: collection.type,
trainingType: collection.trainingType,
chunkSize: collection.chunkSize,
fileId: collection.fileId,
rawLink: collection.rawLink,
metadata: collection.metadata,
createTime: collection.createTime,
session
});
const vectorModelData = getVectorModel(collection.datasetId.vectorModel);
const agentModelData = getLLMModel(collection.datasetId.agentModel);
// start load
await reloadCollectionChunks({
collection: {
...newCol.toObject(),
datasetId: collection.datasetId
},
tmbId,
billId,
rawText,
session
});
await mongoSessionRun(async (session) => {
// create training bill
const { billId } = await createTrainingUsage({
teamId: collection.teamId,
tmbId,
appName: 'core.dataset.collection.Sync Collection',
billSource: UsageSourceEnum.training,
vectorModel: vectorModelData.name,
agentModel: agentModelData.name,
session
});
// delete old collection
await delCollectionAndRelatedSources({
collections: [collection],
session
});
// create a collection and delete old
const newCol = await createOneCollection({
teamId: collection.teamId,
tmbId: collection.tmbId,
parentId: collection.parentId,
datasetId: collection.datasetId._id,
name: title || collection.name,
type: collection.type,
trainingType: collection.trainingType,
chunkSize: collection.chunkSize,
fileId: collection.fileId,
rawLink: collection.rawLink,
metadata: collection.metadata,
createTime: collection.createTime,
session
});
jsonRes(res, {
data: DatasetCollectionSyncResultEnum.success
// start load
await reloadCollectionChunks({
collection: {
...newCol.toObject(),
datasetId: collection.datasetId
},
tmbId,
billId,
rawText,
session
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
// delete old collection
await delCollectionAndRelatedSources({
collections: [collection],
session
});
}
});
return DatasetCollectionSyncResultEnum.success;
}
export default NextAPI(handler);
projects/app/src/pages/api/core/dataset/collection/update.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import type { NextApiRequest } from 'next';
import type { UpdateDatasetCollectionParams } from '@/global/core/api/datasetReq.d';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { getCollectionUpdateTime } from '@fastgpt/service/core/dataset/collection/utils';
import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset';
import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
import { NextAPI } from '@/service/middleware/entry';
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { id, parentId, name } = req.body as UpdateDatasetCollectionParams;
async function handler(req: NextApiRequest) {
const { id, parentId, name } = req.body as UpdateDatasetCollectionParams;
if (!id) {
throw new Error('缺少参数');
}
// 凭证校验
await authDatasetCollection({
req,
authToken: true,
authApiKey: true,
collectionId: id,
per: 'w'
});
if (!id) {
return Promise.reject(CommonErrEnum.missingParams);
}
const updateFields: Record<string, any> = {
...(parentId !== undefined && { parentId: parentId || null }),
...(name && { name, updateTime: getCollectionUpdateTime({ name }) })
};
// 凭证校验
await authDatasetCollection({
req,
authToken: true,
authApiKey: true,
collectionId: id,
per: WritePermissionVal
});
await MongoDatasetCollection.findByIdAndUpdate(id, {
$set: updateFields
});
const updateFields: Record<string, any> = {
...(parentId !== undefined && { parentId: parentId || null }),
...(name && { name, updateTime: getCollectionUpdateTime({ name }) })
};
jsonRes(res);
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
await MongoDatasetCollection.findByIdAndUpdate(id, {
$set: updateFields
});
}
export default NextAPI(handler);
projects/app/src/pages/api/core/dataset/create.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import type { NextApiRequest } from 'next';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import type { CreateDatasetParams } from '@/global/core/dataset/api.d';
import { createDefaultCollection } from '@fastgpt/service/core/dataset/collection/controller';
......@@ -8,62 +6,59 @@ import { authUserPer } from '@fastgpt/service/support/permission/user/auth';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { getLLMModel, getVectorModel, getDatasetModel } from '@fastgpt/service/core/ai/model';
import { checkTeamDatasetLimit } from '@fastgpt/service/support/permission/teamLimit';
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
import { NullPermission, WritePermissionVal } from '@fastgpt/global/support/permission/constant';
import { NextAPI } from '@/service/middleware/entry';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const {
parentId,
name,
type = DatasetTypeEnum.dataset,
avatar,
vectorModel = global.vectorModels[0].model,
agentModel = getDatasetModel().model
} = req.body as CreateDatasetParams;
async function handler(req: NextApiRequest) {
const {
parentId,
name,
type = DatasetTypeEnum.dataset,
avatar,
vectorModel = global.vectorModels[0].model,
agentModel = getDatasetModel().model,
defaultPermission = NullPermission
} = req.body as CreateDatasetParams;
// auth
const { teamId, tmbId } = await authUserPer({
req,
authToken: true,
authApiKey: true,
per: WritePermissionVal
});
// check model valid
const vectorModelStore = getVectorModel(vectorModel);
const agentModelStore = getLLMModel(agentModel);
if (!vectorModelStore || !agentModelStore) {
throw new Error('vectorModel or qaModel is invalid');
}
// auth
const { teamId, tmbId } = await authUserPer({
req,
authToken: true,
authApiKey: true,
per: WritePermissionVal
});
// check limit
await checkTeamDatasetLimit(teamId);
// check model valid
const vectorModelStore = getVectorModel(vectorModel);
const agentModelStore = getLLMModel(agentModel);
if (!vectorModelStore || !agentModelStore) {
throw new Error('vectorModel or qaModel is invalid'); // TODO: use enum code
}
const { _id } = await MongoDataset.create({
name,
teamId,
tmbId,
vectorModel,
agentModel,
avatar,
parentId: parentId || null,
type
});
// check limit
await checkTeamDatasetLimit(teamId);
if (type === DatasetTypeEnum.dataset) {
await createDefaultCollection({
datasetId: _id,
teamId,
tmbId
});
}
const { _id } = await MongoDataset.create({
name,
teamId,
tmbId,
vectorModel,
agentModel,
avatar,
parentId: parentId || null,
type,
defaultPermission
});
jsonRes(res, { data: _id });
} catch (err) {
jsonRes(res, {
code: 500,
error: err
if (type === DatasetTypeEnum.dataset) {
await createDefaultCollection({
datasetId: _id,
teamId,
tmbId
});
}
return _id;
}
export default NextAPI(handler);
projects/app/src/pages/api/core/dataset/data/delete.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { authDatasetData } from '@/service/support/permission/auth/dataset';
import type { NextApiRequest } from 'next';
import { authDatasetData } from '@fastgpt/service/support/permission/dataset/auth';
import { deleteDatasetData } from '@/service/core/dataset/data/controller';
import { NextAPI } from '@/service/middleware/entry';
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
async function handler(req: NextApiRequest) {
const { id: dataId } = req.query as {
id: string;
};
if (!dataId) {
throw new Error('dataId is required');
Promise.reject(CommonErrEnum.missingParams);
}
// 凭证校验
const { teamId, datasetData } = await authDatasetData({
const { datasetData } = await authDatasetData({
req,
authToken: true,
authApiKey: true,
dataId,
per: 'w'
per: WritePermissionVal
});
await deleteDatasetData(datasetData);
jsonRes(res, {
data: 'success'
});
return 'success';
}
export default NextAPI(handler);
projects/app/src/pages/api/core/dataset/data/detail.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authDatasetData } from '@/service/support/permission/auth/dataset';
import type { NextApiRequest } from 'next';
import { NextAPI } from '@/service/middleware/entry';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { authDatasetData } from '@fastgpt/service/support/permission/dataset/auth';
export type Response = {
id: string;
......@@ -11,7 +10,7 @@ export type Response = {
source: string;
};
async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
async function handler(req: NextApiRequest) {
const { id: dataId } = req.query as {
id: string;
};
......@@ -22,12 +21,10 @@ async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
authToken: true,
authApiKey: true,
dataId,
per: 'r'
per: ReadPermissionVal
});
jsonRes(res, {
data: datasetData
});
return datasetData;
}
export default NextAPI(handler);
projects/app/src/pages/api/core/dataset/data/insertData.ts
......@@ -2,30 +2,30 @@
insert one data to dataset (immediately insert)
manual input or mark data
*/
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import type { NextApiRequest } from 'next';
import { countPromptTokens } from '@fastgpt/service/common/string/tiktoken/index';
import { getVectorModel } from '@fastgpt/service/core/ai/model';
import { hasSameValue } from '@/service/core/dataset/data/utils';
import { insertData2Dataset } from '@/service/core/dataset/data/controller';
import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset';
import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
import { getCollectionWithDataset } from '@fastgpt/service/core/dataset/controller';
import { pushGenerateVectorUsage } from '@/service/support/wallet/usage/push';
import { InsertOneDatasetDataProps } from '@/global/core/dataset/api';
import { simpleText } from '@fastgpt/global/common/string/tools';
import { checkDatasetLimit } from '@fastgpt/service/support/permission/teamLimit';
import { NextAPI } from '@/service/middleware/entry';
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
async function handler(req: NextApiRequest) {
const { collectionId, q, a, indexes } = req.body as InsertOneDatasetDataProps;
if (!q) {
throw new Error('q is required');
Promise.reject(CommonErrEnum.missingParams);
}
if (!collectionId) {
throw new Error('collectionId is required');
Promise.reject(CommonErrEnum.missingParams);
}
// 凭证校验
......@@ -34,7 +34,7 @@ async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
authToken: true,
authApiKey: true,
collectionId,
per: 'w'
per: WritePermissionVal
});
await checkDatasetLimit({
......@@ -93,9 +93,7 @@ async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
model: vectorModelData.model
});
jsonRes<string>(res, {
data: insertId
});
return insertId;
}
export default NextAPI(handler);
projects/app/src/pages/api/core/dataset/data/list.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import type { DatasetDataListItemType } from '@/global/core/dataset/type.d';
import type { NextApiRequest } from 'next';
import type { GetDatasetDataListProps } from '@/global/core/api/datasetReq';
import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset';
import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { PagingData } from '@/types';
import { replaceRegChars } from '@fastgpt/global/common/string/tools';
import { NextAPI } from '@/service/middleware/entry';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
async function handler(req: NextApiRequest) {
let {
pageNum = 1,
pageSize = 10,
......@@ -25,7 +22,7 @@ async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
authToken: true,
authApiKey: true,
collectionId,
per: 'r'
per: ReadPermissionVal
});
searchText = replaceRegChars(searchText).replace(/'/g, '');
......@@ -50,14 +47,12 @@ async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
MongoDatasetData.countDocuments(match)
]);
jsonRes<PagingData<DatasetDataListItemType>>(res, {
data: {
pageNum,
pageSize,
data,
total
}
});
return {
pageNum,
pageSize,
data,
total
};
}
export default NextAPI(handler);
projects/app/src/pages/api/core/dataset/data/pushData.ts
......@@ -5,11 +5,12 @@ import type {
PushDatasetDataProps,
PushDatasetDataResponse
} from '@fastgpt/global/core/dataset/api.d';
import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset';
import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
import { checkDatasetLimit } from '@fastgpt/service/support/permission/teamLimit';
import { predictDataLimitLength } from '@fastgpt/global/core/dataset/utils';
import { pushDataListToTrainingQueue } from '@fastgpt/service/core/dataset/training/controller';
import { NextAPI } from '@/service/middleware/entry';
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
const body = req.body as PushDatasetDataProps;
......@@ -29,7 +30,7 @@ async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
authToken: true,
authApiKey: true,
collectionId,
per: 'w'
per: WritePermissionVal
});
// auth dataset limit
......
projects/app/src/pages/api/core/dataset/data/update.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import type { NextApiRequest } from 'next';
import { updateData2Dataset } from '@/service/core/dataset/data/controller';
import { authDatasetData } from '@/service/support/permission/auth/dataset';
import { pushGenerateVectorUsage } from '@/service/support/wallet/usage/push';
import { UpdateDatasetDataProps } from '@/global/core/dataset/api';
import { checkDatasetLimit } from '@fastgpt/service/support/permission/teamLimit';
import { NextAPI } from '@/service/middleware/entry';
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
import { authDatasetData } from '@fastgpt/service/support/permission/dataset/auth';
async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
async function handler(req: NextApiRequest) {
const { id, q = '', a, indexes = [] } = req.body as UpdateDatasetDataProps;
// auth data permission
......@@ -23,7 +22,7 @@ async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
authToken: true,
authApiKey: true,
dataId: id,
per: 'w'
per: WritePermissionVal
});
// auth team balance
......@@ -46,8 +45,6 @@ async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
tokens,
model: vectorModel
});
jsonRes(res);
}
export default NextAPI(handler);
projects/app/src/pages/api/core/dataset/delete.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import type { NextApiRequest } from 'next';
import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
import { delDatasetRelevantData } from '@fastgpt/service/core/dataset/controller';
import { findDatasetAndAllChildren } from '@fastgpt/service/core/dataset/controller';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
import { NextAPI } from '@/service/middleware/entry';
import { OwnerPermissionVal } from '@fastgpt/global/support/permission/constant';
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { id: datasetId } = req.query as {
id: string;
};
async function handler(req: NextApiRequest) {
const { id: datasetId } = req.query as {
id: string;
};
if (!datasetId) {
throw new Error('缺少参数');
}
// auth owner
const { teamId } = await authDataset({
req,
authToken: true,
authApiKey: true,
datasetId,
per: 'owner'
});
if (!datasetId) {
return Promise.reject(CommonErrEnum.missingParams);
}
const datasets = await findDatasetAndAllChildren({
teamId,
datasetId
});
// auth owner
const { teamId } = await authDataset({
req,
authToken: true,
authApiKey: true,
datasetId,
per: OwnerPermissionVal
});
// delete all dataset.data and pg data
await mongoSessionRun(async (session) => {
// delete dataset data
await delDatasetRelevantData({ datasets, session });
await MongoDataset.deleteMany(
{
_id: { $in: datasets.map((d) => d._id) }
},
{ session }
);
});
const datasets = await findDatasetAndAllChildren({
teamId,
datasetId
});
jsonRes(res);
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
// delete all dataset.data and pg data
await mongoSessionRun(async (session) => {
// delete dataset data
await delDatasetRelevantData({ datasets, session });
await MongoDataset.deleteMany(
{
_id: { $in: datasets.map((d) => d._id) }
},
{ session }
);
});
}
export default NextAPI(handler);
projects/app/src/pages/api/core/dataset/detail.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { getLLMModel, getVectorModel } from '@fastgpt/service/core/ai/model';
import type { DatasetItemType } from '@fastgpt/global/core/dataset/type.d';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { NextAPI } from '@/service/middleware/entry';
import { DatasetItemType } from '@fastgpt/global/core/dataset/type';
import { ApiRequestProps } from '@fastgpt/service/type/next';
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { id: datasetId } = req.query as {
id: string;
};
type Query = {
id: string;
};
if (!datasetId) {
throw new Error('缺少参数');
}
async function handler(req: ApiRequestProps<Query>): Promise<DatasetItemType> {
const { id: datasetId } = req.query as {
id: string;
};
// 凭证校验
const { dataset, canWrite, isOwner } = await authDataset({
req,
authToken: true,
authApiKey: true,
datasetId,
per: 'r'
});
jsonRes<DatasetItemType>(res, {
data: {
...dataset,
vectorModel: getVectorModel(dataset.vectorModel),
agentModel: getLLMModel(dataset.agentModel),
canWrite,
isOwner
}
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
if (!datasetId) {
return Promise.reject(CommonErrEnum.missingParams);
}
// 凭证校验
const { dataset, permission } = await authDataset({
req,
authToken: true,
authApiKey: true,
datasetId,
per: ReadPermissionVal
});
return {
...dataset,
permission,
vectorModel: getVectorModel(dataset.vectorModel),
agentModel: getLLMModel(dataset.agentModel)
};
}
export default NextAPI(handler);
projects/app/src/pages/api/core/dataset/exportAll.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { responseWriteController } from '@fastgpt/service/common/response';
import { addLog } from '@fastgpt/service/common/system/log';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { findDatasetAndAllChildren } from '@fastgpt/service/core/dataset/controller';
import {
......@@ -9,6 +9,8 @@ import {
updateExportDatasetLimit
} from '@fastgpt/service/support/user/utils';
import { NextAPI } from '@/service/middleware/entry';
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
let { datasetId } = req.query as {
......@@ -16,11 +18,16 @@ async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
};
if (!datasetId || !global.pgClient) {
throw new Error('缺少参数');
return Promise.reject(CommonErrEnum.missingParams);
}
// 凭证校验
const { teamId } = await authDataset({ req, authToken: true, datasetId, per: 'w' });
const { teamId } = await authDataset({
req,
authToken: true,
datasetId,
per: WritePermissionVal
});
await checkExportDatasetLimit({
teamId,
......
projects/app/src/pages/api/core/dataset/file/detail.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authDatasetFile } from '@fastgpt/service/support/permission/auth/dataset';
import { DatasetFileSchema } from '@fastgpt/global/core/dataset/type.d';
import type { NextApiRequest } from 'next';
import { authDatasetFile } from '@fastgpt/service/support/permission/dataset/auth';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { NextAPI } from '@/service/middleware/entry';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
async function handler(req: NextApiRequest) {
const { fileId } = req.query as { fileId: string };
// 凭证校验
const { file } = await authDatasetFile({ req, authToken: true, fileId, per: ReadPermissionVal });
const { fileId } = req.query as { fileId: string };
// 凭证校验
const { file } = await authDatasetFile({ req, authToken: true, fileId, per: 'r' });
jsonRes<DatasetFileSchema>(res, {
data: file
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
return file;
}
export default NextAPI(handler);
projects/app/src/pages/api/core/dataset/file/getPreviewChunks.ts
import type { NextApiResponse } from 'next';
import { authFile } from '@fastgpt/service/support/permission/auth/file';
import { authDatasetFile } from '@fastgpt/service/support/permission/dataset/auth';
import { DatasetSourceReadTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { rawText2Chunks, readDatasetSourceRawText } from '@fastgpt/service/core/dataset/read';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { NextAPI } from '@/service/middleware/entry';
import { ApiRequestProps } from '@fastgpt/service/type/next';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
export type PostPreviewFilesChunksProps = {
type: DatasetSourceReadTypeEnum;
......@@ -21,8 +21,7 @@ export type PreviewChunksResponse = {
}[];
async function handler(
req: ApiRequestProps<PostPreviewFilesChunksProps>,
res: NextApiResponse<any>
req: ApiRequestProps<PostPreviewFilesChunksProps>
): Promise<PreviewChunksResponse> {
const { type, sourceId, chunkSize, customSplitChar, overlapRatio, selector, isQAImport } =
req.body;
......@@ -36,7 +35,13 @@ async function handler(
const { teamId } = await (async () => {
if (type === DatasetSourceReadTypeEnum.fileLocal) {
return authFile({ req, authToken: true, authApiKey: true, fileId: sourceId });
return authDatasetFile({
req,
authToken: true,
authApiKey: true,
fileId: sourceId,
per: ReadPermissionVal
});
}
return authCert({ req, authApiKey: true, authToken: true });
})();
......
projects/app/src/pages/api/core/dataset/list.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import type { NextApiRequest } from 'next';
import type { DatasetListItemType } from '@fastgpt/global/core/dataset/type.d';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { mongoRPermission } from '@fastgpt/global/support/permission/utils';
import { authUserPer } from '@fastgpt/service/support/permission/user/auth';
import { getVectorModel } from '@fastgpt/service/core/ai/model';
import { NextAPI } from '@/service/middleware/entry';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { DatasetPermission } from '@fastgpt/global/support/permission/dataset/controller';
import {
PerResourceTypeEnum,
ReadPermissionVal
} from '@fastgpt/global/support/permission/constant';
import { MongoResourcePermission } from '@fastgpt/service/support/permission/schema';
import { parseParentIdInMongo } from '@fastgpt/global/common/parentFolder/utils';
async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
async function handler(req: NextApiRequest) {
const { parentId, type } = req.query as { parentId?: string; type?: DatasetTypeEnum };
// 凭证校验
const { teamId, tmbId, permission } = await authUserPer({
const {
teamId,
tmbId,
permission: tmbPer
} = await authUserPer({
req,
authToken: true,
authApiKey: true,
per: ReadPermissionVal
});
const datasets = await MongoDataset.find({
...mongoRPermission({ teamId, tmbId, permission }),
...(parentId !== undefined && { parentId: parentId || null }),
...(type && { type })
})
.sort({ updateTime: -1 })
.lean();
const [myDatasets, rpList] = await Promise.all([
MongoDataset.find({
teamId,
...parseParentIdInMongo(parentId),
...(type && { type })
})
.sort({
updateTime: -1
})
.lean(),
MongoResourcePermission.find({
resourceType: PerResourceTypeEnum.dataset,
teamId,
tmbId
}).lean()
]);
const filterDatasets = myDatasets
.map((dataset) => {
const perVal = rpList.find(
(item) => String(item.resourceId) === String(dataset._id)
)?.permission;
const Per = new DatasetPermission({
per: perVal ?? dataset.defaultPermission,
isOwner: String(dataset.tmbId) === tmbId || tmbPer.isOwner
});
return {
...dataset,
permission: Per
};
})
.filter((app) => app.permission.hasReadPer);
const data = await Promise.all(
datasets.map<DatasetListItemType>((item) => ({
filterDatasets.map<DatasetListItemType>((item) => ({
_id: item._id,
parentId: item.parentId,
avatar: item.avatar,
......@@ -36,15 +70,12 @@ async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
intro: item.intro,
type: item.type,
permission: item.permission,
canWrite: permission.hasWritePer,
isOwner: permission.isOwner || String(item.tmbId) === tmbId,
vectorModel: getVectorModel(item.vectorModel)
vectorModel: getVectorModel(item.vectorModel),
defaultPermission: item.defaultPermission
}))
);
jsonRes<DatasetListItemType[]>(res, {
data
});
return data;
}
export default NextAPI(handler);
projects/app/src/pages/api/core/dataset/paths.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import type { NextApiRequest } from 'next';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import type { ParentTreePathItemType } from '@fastgpt/global/common/parentFolder/type.d';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { parentId } = req.query as { parentId: string };
if (!parentId) {
return jsonRes(res, {
data: []
});
}
await authDataset({ req, authToken: true, datasetId: parentId, per: 'r' });
jsonRes<ParentTreePathItemType[]>(res, {
data: await getParents(parentId)
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { NextAPI } from '@/service/middleware/entry';
async function handler(req: NextApiRequest) {
const { parentId } = req.query as { parentId: string };
if (!parentId) {
return [];
}
await authDataset({ req, authToken: true, datasetId: parentId, per: ReadPermissionVal });
return await getParents(parentId);
}
async function getParents(parentId?: string): Promise<ParentTreePathItemType[]> {
......@@ -44,3 +31,5 @@ async function getParents(parentId?: string): Promise<ParentTreePathItemType[]>
return paths;
}
export default NextAPI(handler);
projects/app/src/pages/api/core/dataset/searchTest.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import type { SearchTestProps, SearchTestResponse } from '@/global/core/dataset/api.d';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import type { NextApiRequest } from 'next';
import type { SearchTestProps } from '@/global/core/dataset/api.d';
import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
import { pushGenerateVectorUsage } from '@/service/support/wallet/usage/push';
import { searchDatasetData } from '@fastgpt/service/core/dataset/search/controller';
import { updateApiKeyUsage } from '@fastgpt/service/support/openapi/tools';
......@@ -13,8 +12,10 @@ import {
checkTeamReRankPermission
} from '@fastgpt/service/support/permission/teamLimit';
import { NextAPI } from '@/service/middleware/entry';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
async function handler(req: NextApiRequest) {
const {
datasetId,
text,
......@@ -29,8 +30,9 @@ async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
} = req.body as SearchTestProps;
if (!datasetId || !text) {
throw new Error('缺少参数');
return Promise.reject(CommonErrEnum.missingParams);
}
const start = Date.now();
// auth dataset role
......@@ -39,7 +41,7 @@ async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
authToken: true,
authApiKey: true,
datasetId,
per: 'r'
per: ReadPermissionVal
});
// auth balance
await checkTeamAIPoints(teamId);
......@@ -88,14 +90,12 @@ async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
});
}
jsonRes<SearchTestResponse>(res, {
data: {
list: searchRes,
duration: `${((Date.now() - start) / 1000).toFixed(3)}s`,
queryExtensionModel: aiExtensionResult?.model,
...result
}
});
return {
list: searchRes,
duration: `${((Date.now() - start) / 1000).toFixed(3)}s`,
queryExtensionModel: aiExtensionResult?.model,
...result
};
}
export default NextAPI(handler);
projects/app/src/pages/api/core/dataset/training/getDatasetTrainingQueue.ts
import type { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next';
import type { ApiRequestProps } from '@fastgpt/service/type/next';
import { NextAPI } from '@/service/middleware/entry';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
type Props = {};
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
export type getDatasetTrainingQueueResponse = {
rebuildingCount: number;
......@@ -12,8 +11,7 @@ export type getDatasetTrainingQueueResponse = {
};
async function handler(
req: ApiRequestProps<any, { datasetId: string }>,
res: ApiResponseType<any>
req: ApiRequestProps<any, { datasetId: string }>
): Promise<getDatasetTrainingQueueResponse> {
const { datasetId } = req.query;
......@@ -22,7 +20,7 @@ async function handler(
authToken: true,
authApiKey: true,
datasetId,
per: 'r'
per: ReadPermissionVal
});
const [rebuildingCount, trainingCount] = await Promise.all([
......
projects/app/src/pages/api/core/dataset/training/getQueueLen.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import type { NextApiRequest } from 'next';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { GetTrainingQueueProps } from '@/global/core/dataset/api';
import { NextAPI } from '@/service/middleware/entry';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
await authCert({ req, authToken: true });
const { vectorModel, agentModel } = req.query as GetTrainingQueueProps;
async function handler(req: NextApiRequest) {
await authCert({ req, authToken: true });
const { vectorModel, agentModel } = req.query as GetTrainingQueueProps;
// get queue data
// 分别统计 model = vectorModel和agentModel的数量
const data = await MongoDatasetTraining.aggregate([
{
$match: {
lockTime: { $lt: new Date('2040/1/1') },
$or: [{ model: { $eq: vectorModel } }, { model: { $eq: agentModel } }]
}
},
{
$group: {
_id: '$model',
count: { $sum: 1 }
}
// get queue data
// 分别统计 model = vectorModel和agentModel的数量
const data = await MongoDatasetTraining.aggregate([
{
$match: {
lockTime: { $lt: new Date('2040/1/1') },
$or: [{ model: { $eq: vectorModel } }, { model: { $eq: agentModel } }]
}
]);
},
{
$group: {
_id: '$model',
count: { $sum: 1 }
}
}
]);
const vectorTrainingCount = data.find((item) => item._id === vectorModel)?.count || 0;
const agentTrainingCount = data.find((item) => item._id === agentModel)?.count || 0;
const vectorTrainingCount = data.find((item) => item._id === vectorModel)?.count || 0;
const agentTrainingCount = data.find((item) => item._id === agentModel)?.count || 0;
jsonRes(res, {
data: {
vectorTrainingCount,
agentTrainingCount
}
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
return {
vectorTrainingCount,
agentTrainingCount
};
}
export default NextAPI(handler);
projects/app/src/pages/api/core/dataset/training/rebuildEmbedding.ts
import { NextAPI } from '@/service/middleware/entry';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
......@@ -8,7 +8,8 @@ import { createTrainingUsage } from '@fastgpt/service/support/wallet/usage/contr
import { UsageSourceEnum } from '@fastgpt/global/support/wallet/usage/constants';
import { getLLMModel, getVectorModel } from '@fastgpt/service/core/ai/model';
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
import { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next';
import { ApiRequestProps } from '@fastgpt/service/type/next';
import { OwnerPermissionVal } from '@fastgpt/global/support/permission/constant';
export type rebuildEmbeddingBody = {
datasetId: string;
......@@ -17,10 +18,7 @@ export type rebuildEmbeddingBody = {
export type Response = {};
async function handler(
req: ApiRequestProps<rebuildEmbeddingBody>,
res: ApiResponseType<any>
): Promise<Response> {
async function handler(req: ApiRequestProps<rebuildEmbeddingBody>): Promise<Response> {
const { datasetId, vectorModel } = req.body;
const { teamId, tmbId, dataset } = await authDataset({
......@@ -28,7 +26,7 @@ async function handler(
authToken: true,
authApiKey: true,
datasetId,
per: 'owner'
per: OwnerPermissionVal
});
// check vector model
......
projects/app/src/pages/api/core/dataset/update.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import type { NextApiRequest } from 'next';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import type { DatasetUpdateBody } from '@fastgpt/global/core/dataset/api.d';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
import { NextAPI } from '@/service/middleware/entry';
import {
OwnerPermissionVal,
WritePermissionVal
} from '@fastgpt/global/support/permission/constant';
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const {
id,
parentId,
name,
avatar,
intro,
permission,
agentModel,
websiteConfig,
externalReadUrl,
status
} = req.body as DatasetUpdateBody;
async function handler(req: NextApiRequest) {
const {
id,
parentId,
name,
avatar,
intro,
agentModel,
websiteConfig,
externalReadUrl,
defaultPermission,
status
} = req.body as DatasetUpdateBody;
if (!id) {
throw new Error('缺少参数');
}
if (!id) {
return Promise.reject(CommonErrEnum.missingParams);
}
if (permission) {
await authDataset({ req, authToken: true, datasetId: id, per: 'owner' });
} else {
await authDataset({ req, authToken: true, datasetId: id, per: 'w' });
}
if (defaultPermission) {
await authDataset({ req, authToken: true, datasetId: id, per: OwnerPermissionVal });
} else {
await authDataset({ req, authToken: true, datasetId: id, per: WritePermissionVal });
}
await MongoDataset.findOneAndUpdate(
{
_id: id
},
{
...(parentId !== undefined && { parentId: parentId || null }),
...(name && { name }),
...(avatar && { avatar }),
...(permission && { permission }),
...(agentModel && { agentModel: agentModel.model }),
...(websiteConfig && { websiteConfig }),
...(status && { status }),
...(intro && { intro }),
...(externalReadUrl && { externalReadUrl })
}
);
console.log('update dataset', req.body);
jsonRes(res);
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
await MongoDataset.findOneAndUpdate(
{
_id: id
},
{
...(parentId !== undefined && { parentId: parentId || null }),
...(name && { name }),
...(avatar && { avatar }),
...(agentModel && { agentModel: agentModel.model }),
...(websiteConfig && { websiteConfig }),
...(status && { status }),
...(intro && { intro }),
...(externalReadUrl && { externalReadUrl }),
defaultPermission
}
);
}
export default NextAPI(handler);
projects/app/src/pages/api/support/user/team/limit/exportDatasetLimit.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import type { NextApiRequest } from 'next';
import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
import { checkExportDatasetLimit } from '@fastgpt/service/support/user/utils';
import { NextAPI } from '@/service/middleware/entry';
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
async function handler(req: NextApiRequest) {
const { datasetId } = req.query as {
datasetId: string;
};
......@@ -13,7 +14,12 @@ async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
}
// 凭证校验
const { teamId } = await authDataset({ req, authToken: true, datasetId, per: 'w' });
const { teamId } = await authDataset({
req,
authToken: true,
datasetId,
per: WritePermissionVal
});
await checkExportDatasetLimit({
teamId,
......
projects/app/src/pages/api/support/wallet/usage/createTrainingUsage.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { UsageSourceEnum } from '@fastgpt/global/support/wallet/usage/constants';
import { CreateTrainingUsageProps } from '@fastgpt/global/support/wallet/usage/api.d';
import { getLLMModel, getVectorModel } from '@fastgpt/service/core/ai/model';
import { createTrainingUsage } from '@fastgpt/service/support/wallet/usage/controller';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
import { NextAPI } from '@/service/middleware/entry';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { name, datasetId } = req.body as CreateTrainingUsageProps;
async function handler(req: NextApiRequest) {
const { name, datasetId } = req.body as CreateTrainingUsageProps;
const { teamId, tmbId, dataset } = await authDataset({
req,
authToken: true,
authApiKey: true,
datasetId,
per: 'w'
});
const { teamId, tmbId, dataset } = await authDataset({
req,
authToken: true,
authApiKey: true,
datasetId,
per: WritePermissionVal
});
const { billId } = await createTrainingUsage({
teamId,
tmbId,
appName: name,
billSource: UsageSourceEnum.training,
vectorModel: getVectorModel(dataset.vectorModel).name,
agentModel: getLLMModel(dataset.agentModel).name
});
const { billId } = await createTrainingUsage({
teamId,
tmbId,
appName: name,
billSource: UsageSourceEnum.training,
vectorModel: getVectorModel(dataset.vectorModel).name,
agentModel: getLLMModel(dataset.agentModel).name
});
jsonRes<string>(res, {
data: billId
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
return billId;
}
export default NextAPI(handler);
projects/app/src/pages/app/list/components/context.tsx
......@@ -14,7 +14,6 @@ import { AppUpdateParams } from '@/global/core/app/api';
import dynamic from 'next/dynamic';
import { useI18n } from '@/web/context/I18n';
import { AppTypeEnum } from '@fastgpt/global/core/app/constants';
const MoveModal = dynamic(() => import('@/components/common/folder/MoveModal'));
type AppListContextType = {
......
projects/app/src/pages/dataset/component/EditFolderModal.tsx
......@@ -36,7 +36,7 @@ const EditFolderModal = ({
if (!val) return Promise.resolve('');
return editCallback(val);
},
onSuccess: (res) => {
onSuccess: () => {
onClose();
}
});
......
projects/app/src/pages/dataset/component/MemberManager.tsx 0 → 100644
import { Box, Button, Flex } from '@chakra-ui/react';
import React from 'react';
import CollaboratorContextProvider, {
MemberManagerInputPropsType
} from '@/components/support/permission/MemberManager/context';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { useTranslation } from 'next-i18next';
function MemberManager({ managePer }: { managePer: MemberManagerInputPropsType }) {
const { t } = useTranslation();
return (
<Box mt={4}>
<CollaboratorContextProvider {...managePer}>
{({ MemberListCard, onOpenManageModal, onOpenAddMember }) => {
return (
<>
<Flex alignItems="center" flexDirection="row" justifyContent="space-between" w="full">
<Flex flexDirection="row" gap="2">
<Button
size="sm"
variant="whitePrimary"
leftIcon={<MyIcon w="4" name="common/settingLight" />}
onClick={onOpenManageModal}
>
{t('permission.Manage')}
</Button>
<Button
size="sm"
variant="whitePrimary"
leftIcon={<MyIcon w="4" name="support/permission/collaborator" />}
onClick={onOpenAddMember}
>
{t('common.Add')}
</Button>
</Flex>
</Flex>
<MemberListCard mt={2} p={1.5} bg="myGray.100" borderRadius="md" />
</>
);
}}
</CollaboratorContextProvider>
</Box>
);
}
export default MemberManager;
projects/app/src/pages/dataset/detail/components/CollectionCard/Header.tsx
......@@ -26,8 +26,6 @@ import EditFolderModal, { useEditFolder } from '../../../component/EditFolderMod
import { TabEnum } from '../../index';
import ParentPath from '@/components/common/ParentPaths';
import dynamic from 'next/dynamic';
import { useUserStore } from '@/web/support/user/useUserStore';
import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant';
import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants';
import { useContextSelector } from 'use-context-selector';
......@@ -40,7 +38,6 @@ const Header = ({}: {}) => {
const { t } = useTranslation();
const theme = useTheme();
const { setLoading } = useSystemStore();
const { userInfo } = useUserStore();
const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
const router = useRouter();
......@@ -189,7 +186,7 @@ const Header = ({}: {}) => {
)}
{/* diff collection button */}
{userInfo?.team?.role !== TeamMemberRoleEnum.visitor && (
{datasetDetail.permission.hasWritePer && (
<>
{datasetDetail?.type === DatasetTypeEnum.dataset && (
<MyMenu
......
projects/app/src/pages/dataset/detail/components/CollectionCard/index.tsx
......@@ -37,8 +37,6 @@ import { useDrag } from '@/web/common/hooks/useDrag';
import SelectCollections from '@/web/core/dataset/components/SelectCollections';
import { useToast } from '@fastgpt/web/hooks/useToast';
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
import { useUserStore } from '@/web/support/user/useUserStore';
import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant';
import { DatasetCollectionSyncResultEnum } from '@fastgpt/global/core/dataset/constants';
import MyBox from '@fastgpt/web/components/common/MyBox';
import { useContextSelector } from 'use-context-selector';
......@@ -53,7 +51,6 @@ const CollectionCard = () => {
const router = useRouter();
const { toast } = useToast();
const { t } = useTranslation();
const { userInfo } = useUserStore();
const { datasetDetail, loadDatasetDetail } = useContextSelector(DatasetPageContext, (v) => v);
const { openConfirm: openDeleteConfirm, ConfirmModal: ConfirmDeleteModal } = useConfirm({
......@@ -213,7 +210,7 @@ const CollectionCard = () => {
}
bg={dragTargetId === collection._id ? 'primary.100' : ''}
userSelect={'none'}
onDragStart={(e) => {
onDragStart={() => {
setDragStartId(collection._id);
}}
onDragOver={(e) => {
......@@ -296,7 +293,7 @@ const CollectionCard = () => {
</Box>
</Td>
<Td onClick={(e) => e.stopPropagation()}>
{collection.canWrite && userInfo?.team?.role !== TeamMemberRoleEnum.visitor && (
{collection.permission.hasWritePer && (
<MyMenu
width={100}
offset={[-70, 5]}
......
projects/app/src/pages/dataset/detail/components/DataCard.tsx
......@@ -35,8 +35,6 @@ import InputDataModal from '../components/InputDataModal';
import RawSourceBox from '@/components/core/dataset/RawSourceBox';
import type { DatasetDataListItemType } from '@/global/core/dataset/type.d';
import { TabEnum } from '..';
import { useUserStore } from '@/web/support/user/useUserStore';
import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import { DatasetCollectionTypeMap, TrainingTypeMap } from '@fastgpt/global/core/dataset/constants';
import { formatTime2YMDHM } from '@fastgpt/global/common/string/time';
......@@ -47,18 +45,21 @@ import { usePagination } from '@fastgpt/web/hooks/usePagination';
import { getCollectionSourceData } from '@fastgpt/global/core/dataset/collection/utils';
import { useI18n } from '@/web/context/I18n';
import EmptyTip from '@fastgpt/web/components/common/EmptyTip';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
import { useContextSelector } from 'use-context-selector';
const DataCard = () => {
const BoxRef = useRef<HTMLDivElement>(null);
const theme = useTheme();
const lastSearch = useRef('');
const router = useRouter();
const { userInfo } = useUserStore();
const { isPc } = useSystemStore();
const { collectionId = '', datasetId } = router.query as {
collectionId: string;
datasetId: string;
};
const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
const { Loading, setIsLoading } = useLoading({ defaultLoading: true });
const { t } = useTranslation();
const { datasetT } = useI18n();
......@@ -101,7 +102,7 @@ const DataCard = () => {
getData(1);
lastSearch.current = searchText;
}, 300),
[]
[searchText]
);
// get file info
......@@ -119,10 +120,7 @@ const DataCard = () => {
}
);
const canWrite = useMemo(
() => userInfo?.team?.role !== TeamMemberRoleEnum.visitor && !!collection?.canWrite,
[collection?.canWrite, userInfo?.team?.role]
);
const canWrite = useMemo(() => datasetDetail.permission.hasWritePer, [datasetDetail]);
const metadataList = useMemo(() => {
if (!collection) return [];
......@@ -291,7 +289,7 @@ const DataCard = () => {
gridTemplateColumns={['1fr', 'repeat(2,1fr)', 'repeat(3,1fr)', 'repeat(4,1fr)']}
gridGap={4}
>
{datasetDataList.map((item, index) => (
{datasetDataList.map((item) => (
<Card
key={item._id}
cursor={'pointer'}
......
projects/app/src/pages/dataset/detail/components/Info.tsx
import React, { useState, useMemo } from 'react';
import React from 'react';
import { useRouter } from 'next/router';
import { Box, Flex, Button, IconButton, Input, Textarea, HStack } from '@chakra-ui/react';
import { DeleteIcon } from '@chakra-ui/icons';
......@@ -11,7 +11,6 @@ import type { DatasetItemType } from '@fastgpt/global/core/dataset/type.d';
import Avatar from '@/components/Avatar';
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
import { useTranslation } from 'next-i18next';
import PermissionRadio from '@/components/support/permission/Radio';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import { useRequest } from '@fastgpt/web/hooks/useRequest';
import { MongoImageTypeEnum } from '@fastgpt/global/common/file/image/constants';
......@@ -25,10 +24,21 @@ import MyDivider from '@fastgpt/web/components/common/MyDivider/index';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel';
import DefaultPermissionList from '@/components/support/permission/DefaultPerList';
import {
DatasetDefaultPermission,
DatasetPermissionList
} from '@fastgpt/global/support/permission/dataset/constant';
import MemberManager from '../../component/MemberManager';
import {
getCollaboratorList,
postUpdateDatasetCollaborators,
deleteDatasetCollaborators
} from '@/web/core/dataset/api/collaborator';
const Info = ({ datasetId }: { datasetId: string }) => {
const { t } = useTranslation();
const { datasetT } = useI18n();
const { datasetT, commonT } = useI18n();
const { datasetDetail, loadDatasetDetail, updateDataset, rebuildingCount, trainingCount } =
useContextSelector(DatasetPageContext, (v) => v);
......@@ -44,7 +54,7 @@ const Info = ({ datasetId }: { datasetId: string }) => {
const avatar = watch('avatar');
const vectorModel = watch('vectorModel');
const agentModel = watch('agentModel');
const permission = watch('permission');
const defaultPermission = watch('defaultPermission');
const { datasetModelList, vectorModelList } = useSystemStore();
......@@ -233,20 +243,46 @@ const Info = ({ datasetId }: { datasetId: string }) => {
<FormLabel flex={['0 0 90px', '0 0 160px']}>{t('common.Intro')}</FormLabel>
<Textarea flex={[1, '0 0 320px']} {...register('intro')} placeholder={t('common.Intro')} />
</Flex>
{datasetDetail.isOwner && (
<Flex mt={5} alignItems={'center'} w={'100%'} flexWrap={'wrap'}>
<FormLabel flex={['0 0 90px', '0 0 160px']} w={0}>
{t('user.Permission')}
</FormLabel>
<Box>
<PermissionRadio
value={permission}
onChange={(e) => {
setValue('permission', e);
}}
{datasetDetail.permission.hasManagePer && (
<>
<Flex mt={5} alignItems={'center'} w={'100%'} flexWrap={'wrap'} maxW="500px">
<FormLabel flex={['0 0 90px', '0 0 160px']} w={0}>
{commonT('permission.Default permission')}
</FormLabel>
<DefaultPermissionList
w="320px"
per={defaultPermission}
defaultPer={DatasetDefaultPermission}
onChange={(v) => setValue('defaultPermission', v)}
/>
</Box>
</Flex>
</Flex>
<Flex mt={5} alignItems={'center'} w={'100%'} flexWrap={'wrap'} maxW="500px">
<FormLabel flex={['0 0 90px', '0 0 160px']} w={0}>
{commonT('permission.Collaborator')}
</FormLabel>
<Box flex={1}>
<MemberManager
managePer={{
permission: datasetDetail.permission,
onGetCollaboratorList: () => getCollaboratorList(datasetId),
permissionList: DatasetPermissionList,
onUpdateCollaborators: (body) =>
postUpdateDatasetCollaborators({
...body,
datasetId
}),
onDelOneCollaborator: (tmbId) =>
deleteDatasetCollaborators({
datasetId,
tmbId
})
}}
/>
</Box>
</Flex>
</>
)}
<Flex mt={5} w={'100%'} alignItems={'flex-end'}>
......@@ -259,7 +295,7 @@ const Info = ({ datasetId }: { datasetId: string }) => {
>
{t('common.Save')}
</Button>
{datasetDetail.isOwner && (
{datasetDetail.permission.isOwner && (
<IconButton
isLoading={btnLoading}
icon={<DeleteIcon />}
......
projects/app/src/pages/dataset/detail/components/InputDataModal.tsx
......@@ -365,9 +365,11 @@ const InputDataModal = ({
<Button variant={'whiteBase'} mr={3} onClick={onClose}>
{t('common.Close')}
</Button>
<MyTooltip label={collection.canWrite ? '' : t('dataset.data.Can not edit')}>
<MyTooltip
label={collection.permission.hasWritePer ? '' : t('dataset.data.Can not edit')}
>
<Button
isDisabled={!collection.canWrite}
isDisabled={!collection.permission.hasWritePer}
// @ts-ignore
onClick={handleSubmit(dataId ? onUpdateData : sureImportData)}
>
......
projects/app/src/pages/dataset/detail/components/Slider.tsx
import React, { useCallback } from 'react';
import { useTranslation } from 'next-i18next';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import { useUserStore } from '@/web/support/user/useUserStore';
import { Box, Flex, IconButton, useTheme, Progress } from '@chakra-ui/react';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import Avatar from '@/components/Avatar';
......@@ -29,7 +27,6 @@ const Slider = ({ currentTab }: { currentTab: TabEnum }) => {
const { datasetT } = useI18n();
const router = useRouter();
const query = router.query;
const { userInfo } = useUserStore();
const { isPc } = useSystemStore();
const { datasetDetail, vectorTrainingMap, agentTrainingMap, rebuildingCount } =
useContextSelector(DatasetPageContext, (v) => v);
......@@ -41,7 +38,7 @@ const Slider = ({ currentTab }: { currentTab: TabEnum }) => {
icon: 'common/overviewLight'
},
{ label: t('core.dataset.test.Search Test'), id: TabEnum.test, icon: 'kbTest' },
...(userInfo?.team.permission.hasManagePer || datasetDetail.isOwner
...(datasetDetail.permission.hasManagePer
? [{ label: t('common.Config'), id: TabEnum.info, icon: 'common/settingLight' }]
: [])
];
......
projects/app/src/pages/dataset/list/component/CreateModal.tsx
import React, { useCallback, useState } from 'react';
import React, { useCallback } from 'react';
import { Box, Flex, Button, ModalFooter, ModalBody, Input } from '@chakra-ui/react';
import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
import { useForm } from 'react-hook-form';
......@@ -20,6 +20,7 @@ import { MongoImageTypeEnum } from '@fastgpt/global/common/file/image/constants'
import AIModelSelector from '@/components/Select/AIModelSelector';
import { useI18n } from '@/web/context/I18n';
import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
import { DatasetDefaultPermission } from '@fastgpt/global/support/permission/dataset/constant';
const CreateModal = ({ onClose, parentId }: { onClose: () => void; parentId?: string }) => {
const { t } = useTranslation();
......@@ -38,7 +39,8 @@ const CreateModal = ({ onClose, parentId }: { onClose: () => void; parentId?: st
name: '',
intro: '',
vectorModel: filterNotHiddenVectorModelList[0].model,
agentModel: datasetModelList[0].model
agentModel: datasetModelList[0].model,
defaultPermission: DatasetDefaultPermission
}
});
const avatar = watch('avatar');
......
projects/app/src/pages/dataset/list/context.tsx 0 → 100644
import { getDatasetPaths } from '@/web/core/dataset/api';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { ParentTreePathItemType } from '@fastgpt/global/common/parentFolder/type';
import { useToast } from '@fastgpt/web/hooks/useToast';
import { useQuery } from '@tanstack/react-query';
import { useTranslation } from 'next-i18next';
import { useRouter } from 'next/router';
import React from 'react';
import { createContext } from 'use-context-selector';
export type DatasetContextType = {
refetch: () => void;
isFetching: boolean;
paths: ParentTreePathItemType[];
};
export const DatasetContext = createContext<DatasetContextType>({
refetch: () => {},
isFetching: false,
paths: []
});
function DatasetContextProvider({ children }: { children: React.ReactNode }) {
const router = useRouter();
const { toast } = useToast();
const { t } = useTranslation();
const { parentId } = router.query as { parentId: string };
const { loadMyDatasets } = useDatasetStore();
const { data, refetch, isFetching } = useQuery(
['loadDataset', parentId],
() => {
return Promise.all([loadMyDatasets(parentId), getDatasetPaths(parentId)]);
},
{
onError(err) {
toast({
status: 'error',
title: t(getErrText(err))
});
}
}
);
const paths = data?.[1] || [];
const contextValue = {
refetch,
isFetching,
paths
};
return <DatasetContext.Provider value={contextValue}>{children}</DatasetContext.Provider>;
}
export default DatasetContextProvider;
projects/app/src/service/support/permission/auth/dataset.ts deleted 100644 → 0
import { DatasetDataItemType } from '@fastgpt/global/core/dataset/type';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset';
import { AuthModeType } from '@fastgpt/service/support/permission/type';
/* data permission same of collection */
export async function authDatasetData({
dataId,
...props
}: AuthModeType & {
dataId: string;
}) {
// get mongo dataset.data
const datasetData = await MongoDatasetData.findById(dataId);
if (!datasetData) {
return Promise.reject('core.dataset.error.Data not found');
}
const result = await authDatasetCollection({
...props,
collectionId: datasetData.collectionId
});
const data: DatasetDataItemType = {
id: String(datasetData._id),
teamId: datasetData.teamId,
q: datasetData.q,
a: datasetData.a,
chunkIndex: datasetData.chunkIndex,
indexes: datasetData.indexes,
datasetId: String(datasetData.datasetId),
collectionId: String(datasetData.collectionId),
sourceName: result.collection.name || '',
sourceId: result.collection?.fileId || result.collection?.rawLink,
isOwner: String(datasetData.tmbId) === result.tmbId,
canWrite: result.canWrite
};
return {
...result,
datasetData: data
};
}
projects/app/src/web/core/app/api/collaborator.ts
......@@ -11,5 +11,5 @@ export const getCollaboratorList = (appId: string) =>
export const postUpdateAppCollaborators = (body: UpdateAppCollaboratorBody) =>
POST('/proApi/core/app/collaborator/update', body);
export const deleteAppCollaborators = ({ ...params }: AppCollaboratorDeleteParams) =>
DELETE('/proApi/core/app/collaborator/delete', { ...params });
export const deleteAppCollaborators = (params: AppCollaboratorDeleteParams) =>
DELETE('/proApi/core/app/collaborator/delete', params);
projects/app/src/web/core/dataset/api/collaborator.ts 0 → 100644
import {
UpdateDatasetCollaboratorBody,
DatasetCollaboratorDeleteParams
} from '@fastgpt/global/core/dataset/collaborator';
import { DELETE, GET, POST } from '@/web/common/api/request';
import { CollaboratorItemType } from '@fastgpt/global/support/permission/collaborator';
export const getCollaboratorList = (datasetId: string) =>
GET<CollaboratorItemType[]>('/proApi/core/dataset/collaborator/list', { datasetId });
export const postUpdateDatasetCollaborators = (body: UpdateDatasetCollaboratorBody) =>
POST('/proApi/core/dataset/collaborator/update', body);
export const deleteDatasetCollaborators = ({ ...params }: DatasetCollaboratorDeleteParams) =>
DELETE('/proApi/core/dataset/collaborator/delete', { ...params });
projects/app/src/web/core/dataset/constants.ts
......@@ -8,6 +8,8 @@ import type {
DatasetCollectionItemType,
DatasetItemType
} from '@fastgpt/global/core/dataset/type.d';
import { DatasetDefaultPermission } from '@fastgpt/global/support/permission/dataset/constant';
import { DatasetPermission } from '@fastgpt/global/support/permission/dataset/controller';
export const defaultDatasetDetail: DatasetItemType = {
_id: '',
......@@ -21,11 +23,10 @@ export const defaultDatasetDetail: DatasetItemType = {
name: '',
intro: '',
status: 'active',
permission: 'private',
isOwner: false,
canWrite: false,
permission: new DatasetPermission(),
vectorModel: defaultVectorModels[0],
agentModel: defaultQAModels[0]
agentModel: defaultQAModels[0],
defaultPermission: DatasetDefaultPermission
};
export const defaultCollectionDetail: DatasetCollectionItemType = {
......@@ -44,20 +45,21 @@ export const defaultCollectionDetail: DatasetCollectionItemType = {
name: '',
intro: '',
status: 'active',
permission: 'private',
permission: new DatasetPermission(),
vectorModel: defaultVectorModels[0].model,
agentModel: defaultQAModels[0].model
agentModel: defaultQAModels[0].model,
defaultPermission: DatasetDefaultPermission
},
parentId: '',
name: '',
type: DatasetCollectionTypeEnum.file,
updateTime: new Date(),
canWrite: false,
sourceName: '',
sourceId: '',
createTime: new Date(),
trainingType: TrainingModeEnum.chunk,
chunkSize: 0
chunkSize: 0,
permission: new DatasetPermission()
};
export enum ImportProcessWayEnum {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment