Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit dd8f2744 by Archer Committed by GitHub
Options

Extraction schema (#398)

parent 7db8d3ea
Showing with 1144 additions and 1096 deletions
Dockerfile
......@@ -9,8 +9,7 @@ ARG name
# copy packages and one project
COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./
COPY ./packages ./packages
COPY ./projects/$name/package.json ./projects/$name/package.json
COPY ./projects/$name/pnpm-lock.yaml ./projects/$name/pnpm-lock.yaml
COPY ./projects/$name ./projects/$name
RUN \
[ -f pnpm-lock.yaml ] && pnpm install || \
......@@ -27,10 +26,7 @@ ARG name
# copy common node_modules and one project node_modules
COPY --from=deps /app/node_modules ./node_modules
COPY --from=deps /app/packages ./packages
COPY ./projects/$name ./projects/$name
COPY --from=deps /app/projects/$name/node_modules ./projects/$name/node_modules
COPY pnpm-lock.yaml pnpm-workspace.yaml ./
COPY ./packages ./packages
COPY --from=deps /app/projects/$name ./projects/$name
# Uncomment the following line in case you want to disable telemetry during the build.
ENV NEXT_TELEMETRY_DISABLED 1
......
docSite/content/docs/development/configuration.md
......@@ -23,7 +23,7 @@ weight: 520
"SystemParams": {
"vectorMaxProcess": 15, // 向量生成最大进程，结合数据库性能和 key 来设置
"qaMaxProcess": 15, // QA 生成最大进程，结合数据库性能和 key 来设置
"pgIvfflatProbe": 20 // pg vector 搜索探针。没有设置索引前可忽略，通常 50w 组以上才需要设置。
"pgHNSWEfSearch": 40 // pg vector 索引参数，越大精度高但速度慢
},
"ChatModels": [
{
......
docSite/content/docs/development/intro.md
......@@ -66,7 +66,7 @@ git clone git@github.com:<github_username>/FastGPT.git
- `vectorMaxProcess`: 向量生成最大进程，根据数据库和 key 的并发数来决定，通常单个 120 号，2c4g 服务器设置 10~15。
- `qaMaxProcess`: QA 生成最大进程
- `pgIvfflatProbe`: PostgreSQL vector 搜索探针，没有添加 vector 索引时可忽略。
- `pgHNSWEfSearch`: PostgreSQL vector 索引参数，越大搜索精度越高但是速度越慢，具体可看 pgvector 官方说明。
### 5. 运行
......
docSite/content/docs/installation/docker.md
......@@ -72,7 +72,7 @@ brew install orbstack
依次执行下面命令，创建 FastGPT 文件并拉取`docker-compose.yml`和`config.json`，执行完后目录下会有 2 个文件。
非 Linux 环境或无法访问外网环境，可手动创建一个目录，并下载下面2个链接的文件。
非 Linux 环境或无法访问外网环境，可手动创建一个目录，并下载下面2个链接的文件: [docker-compose.yml](https://github.com/labring/FastGPT/blob/main/files/deploy/fastgpt/docker-compose.yml),[config.json](https://github.com/labring/FastGPT/blob/main/projects/app/data/config.json)
**注意: `docker-compose.yml` 配置文件中 Mongo 为 5.x，部分服务器不支持，需手动更改其镜像版本为 4.4.24**
......
docSite/content/docs/pricing.md
......@@ -30,6 +30,10 @@ weight: 10
| 星火2.0 - 对话 | 0.01 |
| chatglm_pro - 对话 | 0.01 |
| 通义千问 - 对话 | 0.01 |
| 问题分类 | 0.03 |
| 内容提取 | 0.03 |
| 下一步指引 | 0.015 |
{{< /table >}}
{{% alert context="warning" %}}
......
files/deploy/fastgpt/docker-compose.yml
......@@ -2,8 +2,8 @@
version: '3.3'
services:
pg:
image: ankane/pgvector:v0.4.2 # git
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/pgvector:v0.4.2 # 阿里云
image: ankane/pgvector:v0.5.0 # git
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/pgvector:v0.5.0 # 阿里云
container_name: pg
restart: always
ports: # 生产环境建议不要暴露
......@@ -66,8 +66,8 @@ networks:
# version: '3.3'
# services:
# pg:
# image: ankane/pgvector:v0.4.2 # dockerhub
# # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/pgvector:v0.4.2 # 阿里云
# image: ankane/pgvector:v0.5.0 # dockerhub
# # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/pgvector:v0.5.0 # 阿里云
# container_name: pg
# restart: always
# ports: # 生产环境建议不要暴露
......
package.json
......@@ -8,11 +8,11 @@
},
"devDependencies": {
"husky": "^8.0.3",
"i18next": "^23.2.11",
"lint-staged": "^13.2.1",
"next-i18next": "^14.0.0",
"prettier": "^3.0.3",
"react-i18next": "^13.0.2"
"i18next": "^23.2.11",
"react-i18next": "^13.0.2",
"next-i18next": "^14.0.0"
},
"lint-staged": {
"./**/**/*.{ts,tsx,scss}": "npm run format"
......
packages/common/mongo/index.ts 0 → 100644
import mongoose from 'mongoose';
export default mongoose;
export * from 'mongoose';
export const connectionMongo = global.mongodb || mongoose;
packages/common/mongo/init.ts 0 → 100644
import mongoose from './index';
import 'winston-mongodb';
import { createLogger, format, transports } from 'winston';
/**
* connect MongoDB and init data
*/
export async function connectMongo({
beforeHook,
afterHook
}: {
beforeHook?: () => any;
afterHook?: () => any;
}): Promise<void> {
if (global.mongodb) {
return;
}
global.mongodb = mongoose;
beforeHook && (await beforeHook());
// logger
initLogger();
console.log('mongo start connect');
try {
mongoose.set('strictQuery', true);
await mongoose.connect(process.env.MONGODB_URI as string, {
bufferCommands: true,
maxConnecting: Number(process.env.DB_MAX_LINK || 5),
maxPoolSize: Number(process.env.DB_MAX_LINK || 5),
minPoolSize: 2
});
console.log('mongo connected');
afterHook && (await afterHook());
} catch (error) {
console.log('error->', 'mongo connect error', error);
global.mongodb = undefined;
}
}
function initLogger() {
global.logger = createLogger({
transports: [
new transports.MongoDB({
db: process.env.MONGODB_URI as string,
collection: 'server_logs',
options: {
useUnifiedTopology: true
},
cappedSize: 500000000,
tryReconnect: true,
metaKey: 'meta',
format: format.combine(format.timestamp(), format.json())
}),
new transports.Console({
format: format.combine(
format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
format.printf((info) => {
if (info.level === 'error') {
console.log(info.meta);
return `[${info.level.toLocaleUpperCase()}]: ${[info.timestamp]}: ${info.message}`;
}
return `[${info.level.toLocaleUpperCase()}]: ${[info.timestamp]}: ${info.message}${
info.meta ? `: ${JSON.stringify(info.meta)}` : ''
}`;
})
)
})
]
});
}
packages/common/package.json
{
"name": "@fastgpt/common",
"version": "1.0.0"
"version": "1.0.0",
"dependencies": {
"mongoose": "^7.0.2",
"winston": "^3.10.0",
"winston-mongodb": "^5.1.1",
"axios": "^1.5.1"
},
"devDependencies": {
"@types/node": "^20.8.5"
}
}
projects/app/src/service/common/api/plugins.ts → packages/common/plusApi/censor.ts
import { POST } from './request';
export const postTextCensor = (data: { text: string }) =>
POST<{ code?: number; message: string }>('/plugins/censor/text_baidu', data)
POST<{ code?: number; message: string }>('/common/censor/text_baidu', data)
.then((res) => {
if (res?.code === 5000) {
return Promise.reject(res);
......
packages/common/tools/str.ts
import crypto from 'crypto';
export function strIsLink(str?: string) {
if (!str) return false;
if (/^((http|https)?:\/\/|www\.|\/)[^\s/$.?#].[^\s]*$/i.test(str)) return true;
return false;
}
export const hashStr = (psw: string) => {
return crypto.createHash('sha256').update(psw).digest('hex');
};
/* simple text, remove chinese space and extra \n */
export const simpleText = (text: string) => {
text = text.replace(/([\u4e00-\u9fa5])[\s&&[^\n]]+([\u4e00-\u9fa5])/g, '$1$2');
text = text.replace(/\n{2,}/g, '\n');
text = text.replace(/[\s&&[^\n]]{2,}/g, ' ');
text = text.replace(/[\x00-\x08]/g, ' ');
// replace empty \n
let newText = '';
let lastChar = '';
for (let i = 0; i < text.length; i++) {
const currentChar = text[i];
if (currentChar === '\n' && !/[。？！；.?!;]/g.test(lastChar)) {
} else {
newText += currentChar;
}
lastChar = currentChar;
}
return newText;
};
packages/common/type/index.d.ts 0 → 100644
import type { Mongoose } from '../mongo';
import type { Logger } from 'winston';
export type FeConfigsType = {
show_emptyChat?: boolean;
show_register?: boolean;
show_appStore?: boolean;
show_contact?: boolean;
show_git?: boolean;
show_doc?: boolean;
show_pay?: boolean;
show_openai_account?: boolean;
show_promotion?: boolean;
hide_app_flow?: boolean;
openAPIUrl?: string;
systemTitle?: string;
authorText?: string;
googleClientVerKey?: string;
isPlus?: boolean;
oauth?: {
github?: string;
google?: string;
};
limit?: {
exportLimitMinutes?: number;
};
scripts?: { [key: string]: string }[];
};
export type SystemEnvType = {
pluginBaseUrl?: string;
openapiPrefix?: string;
vectorMaxProcess: number;
qaMaxProcess: number;
pgHNSWEfSearch: number;
};
declare global {
var mongodb: Mongoose | undefined;
var logger: Logger;
var feConfigs: FeConfigsType;
var systemEnv: SystemEnvType;
}
packages/core/ai/config.ts
import { UserModelSchema } from '../user/type';
import type { UserModelSchema } from '@fastgpt/support/user/type.d';
import OpenAI from 'openai';
export const openaiBaseUrl = process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1';
......
packages/core/ai/functions/createQuestionGuide.ts
......@@ -10,7 +10,7 @@ export async function createQuestionGuide({
messages: ChatCompletionRequestMessage[];
model: string;
}) {
const ai = getAIApi();
const ai = getAIApi(undefined, 48000);
const data = await ai.chat.completions.create({
model: model,
temperature: 0,
......@@ -25,7 +25,7 @@ export async function createQuestionGuide({
stream: false
});
const answer = data.choices?.[0].message?.content || '';
const answer = data.choices?.[0]?.message?.content || '';
const totalTokens = data.usage?.total_tokens || 0;
const start = answer.indexOf('[');
......
packages/core/dataset/constant.ts
export enum DatasetTypeEnum {
folder = 'folder',
dataset = 'dataset'
}
export const DatasetTypeMap = {
[DatasetTypeEnum.folder]: {
name: 'folder'
},
[DatasetTypeEnum.dataset]: {
name: 'dataset'
}
};
export enum FileStatusEnum {
embedding = 'embedding',
ready = 'ready'
}
export enum DatasetSpecialIdEnum {
manual = 'manual',
mark = 'mark'
......@@ -13,3 +32,5 @@ export const datasetSpecialIdMap = {
}
};
export const datasetSpecialIds: string[] = [DatasetSpecialIdEnum.manual, DatasetSpecialIdEnum.mark];
export const FolderAvatarSrc = '/imgs/files/folder.svg';
projects/app/src/service/models/kb.ts → packages/core/dataset/schema.ts
import { Schema, model, models, Model } from 'mongoose';
import { kbSchema as SchemaType } from '@/types/mongoSchema';
import { KbTypeMap } from '@/constants/dataset';
import { connectionMongo, type Model } from '@fastgpt/common/mongo';
const { Schema, model, models } = connectionMongo;
import { DatasetSchemaType } from './type';
import { DatasetTypeMap } from './constant';
const kbSchema = new Schema({
const DatasetSchema = new Schema({
parentId: {
type: Schema.Types.ObjectId,
ref: 'kb',
......@@ -32,7 +33,7 @@ const kbSchema = new Schema({
},
type: {
type: String,
enum: Object.keys(KbTypeMap),
enum: Object.keys(DatasetTypeMap),
required: true,
default: 'dataset'
},
......@@ -42,4 +43,4 @@ const kbSchema = new Schema({
}
});
export const KB: Model<SchemaType> = models['kb'] || model('kb', kbSchema);
export const MongoDataset: Model<DatasetSchemaType> = models['kb'] || model('kb', DatasetSchema);
packages/core/dataset/type.d.ts 0 → 100644
import { DatasetTypeEnum } from './constant';
export type DatasetSchemaType = {
_id: string;
userId: string;
parentId: string;
updateTime: Date;
avatar: string;
name: string;
vectorModel: string;
tags: string[];
type: `${DatasetTypeEnum}`;
};
packages/core/tsconfig.json
......@@ -16,6 +16,6 @@
"incremental": true,
"baseUrl": "."
},
"include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", "**/*.d.ts"],
"include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", "**/*.d.ts", "../**/*.d.ts"],
"exclude": ["node_modules"]
}
projects/app/src/service/support/openapi/auth.ts → packages/support/openapi/auth.ts
import { ERROR_ENUM } from '@/service/errorCode';
import { updateApiKeyUsedTime } from './index';
import { OpenApi } from './schema';
import { ERROR_ENUM } from '@fastgpt/common/constant/errorCode';
import { updateApiKeyUsedTime } from './tools';
import { MongoOpenApi } from './schema';
import { POST } from '@fastgpt/common/plusApi/request';
import { OpenApiSchema } from './type.d';
export type AuthOpenApiLimitProps = { openApi: OpenApiSchema };
export async function authOpenApiKey({ apikey }: { apikey: string }) {
if (!apikey) {
......@@ -8,7 +12,7 @@ export async function authOpenApiKey({ apikey }: { apikey: string }) {
}
try {
const openApi = await OpenApi.findOne({ apiKey: apikey });
const openApi = await MongoOpenApi.findOne({ apiKey: apikey });
if (!openApi) {
return Promise.reject(ERROR_ENUM.unAuthApiKey);
}
......@@ -16,17 +20,7 @@ export async function authOpenApiKey({ apikey }: { apikey: string }) {
// auth limit
if (global.feConfigs?.isPlus) {
if (openApi?.limit?.expiredTime && openApi.limit.expiredTime.getTime() < Date.now()) {
return Promise.reject(`Key ${openApi.apiKey} is expired`);
}
if (
openApi?.limit?.credit &&
openApi.limit.credit > -1 &&
openApi.usage > openApi.limit.credit
) {
return Promise.reject(`Key ${openApi.apiKey} is over usage`);
}
await POST('/support/openapi/authLimit', { openApi } as AuthOpenApiLimitProps);
}
updateApiKeyUsedTime(openApi._id);
......
projects/app/src/service/support/openapi/schema.ts → packages/support/openapi/schema.ts
import { Schema, model, models, Model } from 'mongoose';
import { OpenApiSchema } from '@/types/support/openapi';
import { connectionMongo, type Model } from '@fastgpt/common/mongo';
const { Schema, model, models } = connectionMongo;
import { OpenApiSchema } from './type.d';
import { PRICE_SCALE } from '@fastgpt/common/bill/constants';
import { formatPrice } from '@fastgpt/common/bill/index';
......@@ -54,4 +55,5 @@ const OpenApiSchema = new Schema(
}
);
export const OpenApi: Model<OpenApiSchema> = models['openapi'] || model('openapi', OpenApiSchema);
export const MongoOpenApi: Model<OpenApiSchema> =
models['openapi'] || model('openapi', OpenApiSchema);
projects/app/src/service/support/openapi/index.ts → packages/support/openapi/tools.ts
import { OpenApi } from './schema';
import { MongoOpenApi } from './schema';
export async function updateApiKeyUsedTime(id: string) {
await OpenApi.findByIdAndUpdate(id, {
await MongoOpenApi.findByIdAndUpdate(id, {
lastUsedTime: new Date()
});
}
export async function updateApiKeyUsage({ apikey, usage }: { apikey: string; usage: number }) {
await OpenApi.findOneAndUpdate(
await MongoOpenApi.findOneAndUpdate(
{ apiKey: apikey },
{
$inc: {
......
packages/support/outLink/auth.ts 0 → 100644
import { AuthUserTypeEnum, authBalanceByUid } from '../user/auth';
import { MongoOutLink } from './schema';
import { POST } from '@fastgpt/common/plusApi/request';
import { OutLinkSchema } from './type.d';
export type AuthLinkProps = { ip?: string | null; authToken?: string; question: string };
export type AuthLinkLimitProps = AuthLinkProps & { outLink: OutLinkSchema };
export async function authOutLinkChat({
shareId,
ip,
authToken,
question
}: AuthLinkProps & {
shareId: string;
}) {
// get outLink
const outLink = await MongoOutLink.findOne({
shareId
});
if (!outLink) {
return Promise.reject('分享链接无效');
}
const uid = String(outLink.userId);
const [user] = await Promise.all([
authBalanceByUid(uid), // authBalance
...(global.feConfigs?.isPlus ? [authOutLinkLimit({ outLink, ip, authToken, question })] : []) // limit auth
]);
return {
user,
userId: String(outLink.userId),
appId: String(outLink.appId),
authType: AuthUserTypeEnum.token,
responseDetail: outLink.responseDetail
};
}
export function authOutLinkLimit(data: AuthLinkLimitProps) {
return POST('/support/outLink/authLimit', data);
}
export async function authOutLinkId({ id }: { id: string }) {
const outLink = await MongoOutLink.findOne({
shareId: id
});
if (!outLink) {
return Promise.reject('分享链接无效');
}
return {
userId: String(outLink.userId)
};
}
export type AuthShareChatInitProps = {
authToken?: string;
tokenUrl?: string;
};
export function authShareChatInit(data: AuthShareChatInitProps) {
return POST('/support/outLink/authShareChatInit', data);
}
packages/support/outLink/constant.ts 0 → 100644
export enum OutLinkTypeEnum {
'share' = 'share',
'iframe' = 'iframe',
apikey = 'apikey'
}
projects/app/src/service/support/outLink/schema.ts → packages/support/outLink/schema.ts
import { Schema, model, models, Model } from 'mongoose';
import { OutLinkSchema as SchemaType } from '@/types/support/outLink';
import { OutLinkTypeEnum } from '@/constants/chat';
import { connectionMongo, type Model } from '@fastgpt/common/mongo';
const { Schema, model, models } = connectionMongo;
import { OutLinkSchema as SchemaType } from './type.d';
import { OutLinkTypeEnum } from './constant';
const OutLinkSchema = new Schema({
shareId: {
......@@ -55,4 +56,5 @@ const OutLinkSchema = new Schema({
}
});
export const OutLink: Model<SchemaType> = models['outlinks'] || model('outlinks', OutLinkSchema);
export const MongoOutLink: Model<SchemaType> =
models['outlinks'] || model('outlinks', OutLinkSchema);
projects/app/src/service/support/outLink/index.ts → packages/support/outLink/tools.ts
import { addLog } from '@/service/utils/tools';
import { ChatHistoryItemResType } from '@/types/chat';
import axios from 'axios';
import { OutLink } from './schema';
import { MongoOutLink } from './schema';
export const updateOutLinkUsage = async ({
shareId,
......@@ -11,7 +9,7 @@ export const updateOutLinkUsage = async ({
total: number;
}) => {
try {
await OutLink.findOneAndUpdate(
await MongoOutLink.findOneAndUpdate(
{ shareId },
{
$inc: { total },
......@@ -19,7 +17,7 @@ export const updateOutLinkUsage = async ({
}
);
} catch (err) {
addLog.error('update shareChat error', err);
console.log('update shareChat error', err);
}
};
......@@ -30,11 +28,11 @@ export const pushResult2Remote = async ({
}: {
authToken?: string;
shareId?: string;
responseData?: ChatHistoryItemResType[];
responseData?: any[];
}) => {
if (!shareId || !authToken) return;
try {
const outLink = await OutLink.findOne({
const outLink = await MongoOutLink.findOne({
shareId
});
if (!outLink?.limit?.hookUrl) return;
......
projects/app/src/types/support/outLink.d.ts → packages/support/outLink/type.d.ts
import { OutLinkTypeEnum } from '@/constants/chat';
import { OutLinkTypeEnum } from './constant';
export interface OutLinkSchema {
export type OutLinkSchema = {
_id: string;
shareId: string;
userId: string;
......@@ -16,7 +16,7 @@ export interface OutLinkSchema {
credit: number;
hookUrl?: string;
};
}
};
export type OutLinkEditType = {
_id?: string;
......
packages/support/package.json
......@@ -2,6 +2,13 @@
"name": "@fastgpt/support",
"version": "1.0.0",
"dependencies": {
"@fastgpt/common": "workspace:*"
"@fastgpt/common": "workspace:*",
"cookie": "^0.5.0",
"jsonwebtoken": "^9.0.2",
"axios": "^1.5.1"
},
"devDependencies": {
"@types/cookie": "^0.5.2",
"@types/jsonwebtoken": "^9.0.3"
}
}
packages/support/tsconfig.json
......@@ -14,8 +14,11 @@
"isolatedModules": true,
"jsx": "preserve",
"incremental": true,
"baseUrl": "."
"baseUrl": ".",
"paths": {
"@/*": ["./*"]
}
},
"include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", "**/*.d.ts"],
"include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", "**/*.d.ts", "../**/*.d.ts"],
"exclude": ["node_modules"]
}
packages/support/user/auth.ts 0 → 100644
import Cookie from 'cookie';
import { authJWT } from './tools';
import { authOpenApiKey } from '../openapi/auth';
import { authOutLinkId } from '../outLink/auth';
import { MongoUser } from './schema';
import type { UserModelSchema } from './type.d';
import { ERROR_ENUM } from '@fastgpt/common/constant/errorCode';
export enum AuthUserTypeEnum {
token = 'token',
root = 'root',
apikey = 'apikey',
outLink = 'outLink'
}
/* auth balance */
export const authBalanceByUid = async (uid: string) => {
const user = await MongoUser.findById<UserModelSchema>(
uid,
'_id username balance openaiAccount timezone'
);
if (!user) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
if (user.balance <= 0) {
return Promise.reject(ERROR_ENUM.insufficientQuota);
}
return user;
};
/* uniform auth user */
export const authUser = async ({
req,
authToken = false,
authRoot = false,
authApiKey = false,
authBalance = false,
authOutLink
}: {
req: any;
authToken?: boolean;
authRoot?: boolean;
authApiKey?: boolean;
authBalance?: boolean;
authOutLink?: boolean;
}) => {
const authCookieToken = async (cookie?: string, token?: string): Promise<string> => {
// 获取 cookie
const cookies = Cookie.parse(cookie || '');
const cookieToken = cookies.token || token;
if (!cookieToken) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
return await authJWT(cookieToken);
};
// from authorization get apikey
const parseAuthorization = async (authorization?: string) => {
if (!authorization) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
// Bearer fastgpt-xxxx-appId
const auth = authorization.split(' ')[1];
if (!auth) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
const { apikey, appId: authorizationAppid = '' } = await (async () => {
const arr = auth.split('-');
// abandon
if (arr.length === 3) {
return {
apikey: `${arr[0]}-${arr[1]}`,
appId: arr[2]
};
}
if (arr.length === 2) {
return {
apikey: auth
};
}
return Promise.reject(ERROR_ENUM.unAuthorization);
})();
// auth apikey
const { userId, appId: apiKeyAppId = '' } = await authOpenApiKey({ apikey });
return {
uid: userId,
apikey,
appId: apiKeyAppId || authorizationAppid
};
};
// root user
const parseRootKey = async (rootKey?: string, userId = '') => {
if (!rootKey || !process.env.ROOT_KEY || rootKey !== process.env.ROOT_KEY) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
return userId;
};
const { cookie, token, apikey, rootkey, userid, authorization } = (req.headers || {}) as {
cookie?: string;
token?: string;
apikey?: string;
rootkey?: string; // abandon
userid?: string;
authorization?: string;
};
const { shareId } = (req?.body || {}) as { shareId?: string };
let uid = '';
let appId = '';
let openApiKey = apikey;
let authType: `${AuthUserTypeEnum}` = AuthUserTypeEnum.token;
if (authOutLink && shareId) {
const res = await authOutLinkId({ id: shareId });
uid = res.userId;
authType = AuthUserTypeEnum.outLink;
} else if (authToken && (cookie || token)) {
// user token(from fastgpt web)
uid = await authCookieToken(cookie, token);
authType = AuthUserTypeEnum.token;
} else if (authRoot && rootkey) {
// root user
uid = await parseRootKey(rootkey, userid);
authType = AuthUserTypeEnum.root;
} else if (authApiKey && apikey) {
// apikey
const parseResult = await authOpenApiKey({ apikey });
uid = parseResult.userId;
authType = AuthUserTypeEnum.apikey;
openApiKey = parseResult.apikey;
} else if (authApiKey && authorization) {
// apikey from authorization
const authResponse = await parseAuthorization(authorization);
uid = authResponse.uid;
appId = authResponse.appId;
openApiKey = authResponse.apikey;
authType = AuthUserTypeEnum.apikey;
}
// not rootUser and no uid, reject request
if (!rootkey && !uid) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
// balance check
const user = await (() => {
if (authBalance) {
return authBalanceByUid(uid);
}
})();
return {
userId: String(uid),
appId,
authType,
user,
apikey: openApiKey
};
};
projects/app/src/service/models/user.ts → packages/support/user/schema.ts
import { Schema, model, models, Model } from 'mongoose';
import { hashPassword } from '@/service/utils/tools';
import { connectionMongo, type Model } from '@fastgpt/common/mongo';
const { Schema, model, models } = connectionMongo;
import { hashStr } from '@fastgpt/common/tools/str';
import { PRICE_SCALE } from '@fastgpt/common/bill/constants';
import { UserModelSchema } from '@/types/mongoSchema';
import type { UserModelSchema } from './type.d';
const UserSchema = new Schema({
username: {
......@@ -13,8 +14,8 @@ const UserSchema = new Schema({
password: {
type: String,
required: true,
set: (val: string) => hashPassword(val),
get: (val: string) => hashPassword(val),
set: (val: string) => hashStr(val),
get: (val: string) => hashStr(val),
select: false
},
createTime: {
......@@ -42,6 +43,9 @@ const UserSchema = new Schema({
exportKbTime: {
// Every half hour
type: Date
},
datasetMaxCount: {
type: Number
}
},
openaiAccount: {
......@@ -56,4 +60,4 @@ const UserSchema = new Schema({
}
});
export const User: Model<UserModelSchema> = models['user'] || model('user', UserSchema);
export const MongoUser: Model<UserModelSchema> = models['user'] || model('user', UserSchema);
packages/support/user/tools.ts 0 → 100644
import jwt from 'jsonwebtoken';
import { ERROR_ENUM } from '@fastgpt/common/constant/errorCode';
/* 生成 token */
export const generateToken = (userId: string) => {
const key = process.env.TOKEN_KEY as string;
const token = jwt.sign(
{
userId,
exp: Math.floor(Date.now() / 1000) + 60 * 60 * 24 * 7
},
key
);
return token;
};
// auth token
export const authJWT = (token: string) =>
new Promise<string>((resolve, reject) => {
const key = process.env.TOKEN_KEY as string;
jwt.verify(token, key, function (err, decoded: any) {
if (err || !decoded?.userId) {
reject(ERROR_ENUM.unAuthorization);
return;
}
resolve(decoded.userId);
});
});
packages/core/user/type.d.ts → packages/support/user/type.d.ts
......@@ -15,5 +15,6 @@ export type UserModelSchema = {
};
limit: {
exportKbTime?: Date;
datasetMaxCount?: number;
};
};
pnpm-lock.yaml
This source diff could not be displayed because it is too large. You can view the blob instead.
projects/app/data/config.json
......@@ -2,7 +2,7 @@
"SystemParams": {
"vectorMaxProcess": 15,
"qaMaxProcess": 15,
"pgIvfflatProbe": 20
"pgHNSWEfSearch ": 40
},
"ChatModels": [
{
......
projects/app/next.config.js
......@@ -5,9 +5,8 @@ const path = require('path');
const nextConfig = {
i18n,
output: 'standalone',
reactStrictMode: false,
reactStrictMode: process.env.NODE_ENV === 'development' ? false : true,
compress: true,
transpilePackages: ['@fastgpt/*'],
webpack(config, { isServer }) {
if (!isServer) {
config.resolve = {
......@@ -18,6 +17,16 @@ const nextConfig = {
}
};
}
Object.assign(config.resolve.alias, {
'@mongodb-js/zstd': false,
'@aws-sdk/credential-providers': false,
snappy: false,
aws4: false,
'mongodb-client-encryption': false,
kerberos: false,
'supports-color': false,
'bson-ext': false
});
config.module = {
...config.module,
rules: config.module.rules.concat([
......@@ -27,12 +36,15 @@ const nextConfig = {
use: ['@svgr/webpack']
}
]),
exprContextCritical: false
exprContextCritical: false,
unknownContextCritical: false
};
return config;
},
transpilePackages: ['@fastgpt/*'],
experimental: {
serverComponentsExternalPackages: ['mongoose'],
outputFileTracingRoot: path.join(__dirname, '../../')
}
};
......
projects/app/package.json
......@@ -22,9 +22,7 @@
"@mozilla/readability": "^0.4.4",
"@tanstack/react-query": "^4.24.10",
"@types/nprogress": "^0.2.0",
"axios": "^1.3.3",
"cookie": "^0.5.0",
"crypto": "^1.0.1",
"axios": "^1.5.1",
"date-fns": "^2.30.0",
"dayjs": "^1.11.7",
"downloadjs": "^1.4.7",
......@@ -33,21 +31,18 @@
"formidable": "^2.1.1",
"framer-motion": "^9.0.6",
"hyperdown": "^2.4.29",
"i18next": "^22.5.1",
"immer": "^9.0.19",
"js-cookie": "^3.0.5",
"js-tiktoken": "^1.0.7",
"jschardet": "^3.0.0",
"jsdom": "^22.1.0",
"jsonwebtoken": "^9.0.0",
"jsonwebtoken": "^9.0.2",
"lodash": "^4.17.21",
"mammoth": "^1.5.1",
"mermaid": "^10.2.3",
"mongoose": "^6.10.0",
"multer": "1.4.5-lts.1",
"nanoid": "^4.0.1",
"next": "13.5.2",
"next-i18next": "^14.0.3",
"nextjs-cors": "^2.1.2",
"nprogress": "^0.2.0",
"papaparse": "^5.4.1",
......@@ -57,7 +52,6 @@
"react-day-picker": "^8.7.1",
"react-dom": "18.2.0",
"react-hook-form": "^7.43.1",
"react-i18next": "^12.3.1",
"react-markdown": "^8.0.7",
"react-syntax-highlighter": "^15.5.0",
"reactflow": "^11.7.4",
......@@ -68,21 +62,20 @@
"request-ip": "^3.3.0",
"sass": "^1.58.3",
"timezones-list": "^3.0.2",
"winston": "^3.10.0",
"winston-mongodb": "^5.1.1",
"zustand": "^4.3.5"
"zustand": "^4.3.5",
"i18next": "^23.2.11",
"react-i18next": "^13.0.2",
"next-i18next": "^14.0.0"
},
"devDependencies": {
"@svgr/webpack": "^6.5.1",
"@types/cookie": "^0.5.1",
"@types/downloadjs": "^1.4.3",
"@types/formidable": "^2.0.5",
"@types/js-cookie": "^3.0.3",
"@types/jsdom": "^21.1.1",
"@types/jsonwebtoken": "^9.0.1",
"@types/jsonwebtoken": "^9.0.3",
"@types/lodash": "^4.14.191",
"@types/multer": "^1.4.7",
"@types/node": "18.14.0",
"@types/papaparse": "^5.3.7",
"@types/pg": "^8.6.6",
"@types/react": "18.0.28",
......
projects/app/pnpm-lock.yaml deleted 100644 → 0
This source diff could not be displayed because it is too large. You can view the blob instead.
projects/app/public/locales/en/common.json
......@@ -181,6 +181,7 @@
"Choice QA Desc": "The index is constructed with the structure of QA pairs, and ADAPTS to various scenarios such as Q&A and reading",
"Choice Visual": "Visual workflow",
"Choice Visual Desc": "Visualize modular operations, easily implement complex workflows, and make your AI no longer monolithic",
"Commercial": "Commercial",
"Community": "Community",
"Dateset": "",
"Dateset Desc": "",
......
projects/app/public/locales/zh/common.json
......@@ -181,6 +181,7 @@
"Choice QA Desc": "采用 QA 对的结构构建索引，适应问答、阅读等多种场景",
"Choice Visual": "可视化工作流",
"Choice Visual Desc": "可视化模块操作，轻松实现复杂工作流，让你的 AI 不再单一",
"Commercial": "商务咨询",
"Community": "社区",
"Dateset": "自动数据预处理",
"Dateset Desc": "提供手动输入、直接分段、LLM 自动处理和 CSV 等多种数据导入途径",
......
projects/app/src/components/ChatBox/SelectDataset.tsx
......@@ -4,7 +4,7 @@ import { useTranslation } from 'next-i18next';
import { useToast } from '@/web/common/hooks/useToast';
import Avatar from '../Avatar';
import MyIcon from '@/components/Icon';
import { KbTypeEnum } from '@/constants/dataset';
import { DatasetTypeEnum } from '@fastgpt/core/dataset/constant';
import DatasetSelectModal, { useDatasetSelect } from '@/components/core/dataset/SelectModal';
const SelectDataset = ({
......@@ -57,7 +57,7 @@ const SelectDataset = ({
}
: {})}
onClick={() => {
if (item.type === KbTypeEnum.folder) {
if (item.type === DatasetTypeEnum.folder) {
setParentId(item._id);
} else {
setSelectedId(item._id);
......
projects/app/src/components/MyModal/index.tsx
......@@ -45,8 +45,8 @@ const MyModal = ({
{...props}
>
{!!title && <ModalHeader>{title}</ModalHeader>}
{onClose && <ModalCloseButton />}
<Box overflow={'overlay'} h={'100%'}>
{onClose && <ModalCloseButton />}
{children}
</Box>
</ModalContent>
......
projects/app/src/constants/chat.ts
......@@ -53,11 +53,5 @@ export const ChatSourceMap = {
}
};
export enum OutLinkTypeEnum {
'share' = 'share',
'iframe' = 'iframe',
apikey = 'apikey'
}
export const HUMAN_ICON = `/icon/human.svg`;
export const LOGO_ICON = `/icon/logo.svg`;
projects/app/src/constants/dataset.ts
......@@ -14,23 +14,3 @@ export const defaultKbDetail: DatasetItemType = {
maxToken: 3000
}
};
export enum KbTypeEnum {
folder = 'folder',
dataset = 'dataset'
}
export enum FileStatusEnum {
embedding = 'embedding',
ready = 'ready'
}
export const KbTypeMap = {
[KbTypeEnum.folder]: {
name: 'folder'
},
[KbTypeEnum.dataset]: {
name: 'dataset'
}
};
export const FolderAvatarSrc = '/imgs/files/folder.svg';
projects/app/src/constants/flow/index.ts
......@@ -2,7 +2,7 @@ import type { BoxProps } from '@chakra-ui/react';
export enum FlowInputItemTypeEnum {
systemInput = 'systemInput', // history, userChatInput, variableInput
input = 'input',
input = 'input', // one line input
textarea = 'textarea',
numberInput = 'numberInput',
select = 'select',
......@@ -10,6 +10,7 @@ export enum FlowInputItemTypeEnum {
custom = 'custom',
target = 'target', // data input
switch = 'switch',
chatInput = 'chatInput',
selectApp = 'selectApp',
// chat special input
quoteList = 'quoteList',
......
projects/app/src/constants/model.ts
import type { AppSchema } from '@/types/mongoSchema';
import type { OutLinkEditType } from '@/types/support/outLink';
import type { OutLinkEditType } from '@fastgpt/support/outLink/type.d';
export const defaultApp: AppSchema = {
_id: '',
......
projects/app/src/global/common/api/systemRes.d.ts
......@@ -4,11 +4,13 @@ import {
type VectorModelItemType,
FunctionModelItemType
} from '@/types/model';
import type { FeConfigsType } from '@fastgpt/common/type/index.d';
export type InitDateResponse = {
chatModels: ChatModelItemType[];
qaModel: QAModelItemType;
vectorModels: VectorModelItemType[];
feConfigs: FeConfigsType;
priceMd: string;
systemVersion: string;
};
projects/app/src/global/core/api/datasetReq.d.ts
import { KbTypeEnum } from '@/constants/dataset';
import { DatasetTypeEnum } from '@fastgpt/core/dataset/constant';
import type { RequestPaging } from '@/types';
import { TrainingModeEnum } from '@/constants/plugin';
import type { SearchTestItemType } from '@/types/core/dataset';
......@@ -18,7 +18,7 @@ export type CreateDatasetParams = {
tags: string[];
avatar: string;
vectorModel?: string;
type: `${KbTypeEnum}`;
type: `${DatasetTypeEnum}`;
};
export type SearchTestProps = {
......
projects/app/src/global/core/api/datasetRes.d.ts
import { KbTypeEnum } from '@/constants/dataset';
import type { RequestPaging } from '@/types';
import { TrainingModeEnum } from '@/constants/plugin';
import type { SearchTestItemType } from '@/types/core/dataset';
......
projects/app/src/global/support/api/openapiReq.d.ts
import type { OpenApiSchema } from '@/types/support/openapi';
import type { OpenApiSchema } from '@fastgpt/support/openapi/type.d';
export type GetApiKeyProps = {
appId?: string;
......
projects/app/src/pages/_app.tsx
......@@ -13,10 +13,10 @@ import { appWithTranslation, useTranslation } from 'next-i18next';
import { getLangStore, setLangStore } from '@/web/common/utils/i18n';
import { useRouter } from 'next/router';
import { useGlobalStore } from '@/web/common/store/global';
import type { FeConfigsType } from '@fastgpt/common/type/index.d';
import 'nprogress/nprogress.css';
import '@/web/styles/reset.scss';
import { FeConfigsType } from '@/types';
//Binding events.
Router.events.on('routeChangeStart', () => NProgress.start());
......
projects/app/src/pages/_error.tsx
......@@ -2,8 +2,6 @@ import { useEffect } from 'react';
import { useRouter } from 'next/router';
import { serviceSideProps } from '@/web/common/utils/i18n';
import { useGlobalStore } from '@/web/common/store/global';
import { addLog } from '@/service/utils/tools';
import { getErrText } from '@/utils/tools';
function Error() {
const router = useRouter();
......@@ -36,8 +34,6 @@ function Error() {
export async function getServerSideProps(context: any) {
console.log('[render error]: ', context);
addLog.error(getErrText(context?.res));
return {
props: { ...(await serviceSideProps(context)) }
};
......
projects/app/src/pages/account/components/PayModal.tsx
......@@ -6,10 +6,9 @@ import { useQuery } from '@tanstack/react-query';
import { useRouter } from 'next/router';
import { getErrText } from '@/utils/tools';
import { useTranslation } from 'react-i18next';
import { formatPrice } from '@fastgpt/common/bill/index';
import Markdown from '@/components/Markdown';
import MyModal from '@/components/MyModal';
import { vectorModelList, chatModelList, qaModel } from '@/web/common/store/static';
import { priceMd } from '@/web/common/store/static';
const PayModal = ({ onClose }: { onClose: () => void }) => {
const router = useRouter();
......@@ -70,10 +69,15 @@ const PayModal = ({ onClose }: { onClose: () => void }) => {
title={t('user.Pay')}
isCentered={!payId}
>
<ModalBody py={0}>
<ModalBody
p={0}
h={payId ? 'auto' : ['auto', '70vh']}
display={'flex'}
flexDirection={'column'}
>
{!payId && (
<>
<Grid gridTemplateColumns={'repeat(4,1fr)'} gridGap={5} mb={4}>
<Grid gridTemplateColumns={'repeat(4,1fr)'} gridGap={5} mb={4} px={6}>
{[10, 20, 50, 100].map((item) => (
<Button
key={item}
......@@ -84,7 +88,7 @@ const PayModal = ({ onClose }: { onClose: () => void }) => {
</Button>
))}
</Grid>
<Box mb={4}>
<Box mb={4} px={6}>
<Input
value={inputVal}
type={'number'}
......@@ -95,18 +99,9 @@ const PayModal = ({ onClose }: { onClose: () => void }) => {
}}
></Input>
</Box>
<Markdown
source={`
| 计费项 | 价格: 元/ 1K tokens(包含上下文)|
| --- | --- |
${vectorModelList
.map((item) => `| 索引-${item.name} | ${formatPrice(item.price, 1000)} |`)
.join('\n')}
${chatModelList
.map((item) => `| 对话-${item.name} | ${formatPrice(item.price, 1000)} |`)
.join('\n')}
| 文件QA拆分 | ${formatPrice(qaModel.price, 1000)} |`}
/>
<Box flex={[1, '1 0 0']} overflow={'overlay'} px={6}>
<Markdown source={priceMd} />
</Box>
</>
)}
{/* 付费二维码 */}
......
projects/app/src/pages/api/admin/initv43.ts
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { authUser } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
import { PgClient } from '@/service/pg';
import { PgDatasetTableName } from '@/constants/plugin';
......
projects/app/src/pages/api/admin/initv44.ts
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { authUser } from '@/service/utils/auth';
import { connectToDatabase, KB } from '@/service/mongo';
import { KbTypeEnum } from '@/constants/dataset';
import { authUser } from '@fastgpt/support/user/auth';
import { connectToDatabase } from '@/service/mongo';
import { MongoDataset } from '@fastgpt/core/dataset/schema';
import { DatasetTypeEnum } from '@fastgpt/core/dataset/constant';
import { PgClient } from '@/service/pg';
import { PgDatasetTableName } from '@/constants/plugin';
......@@ -12,13 +13,13 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
await connectToDatabase();
await authUser({ req, authRoot: true });
await KB.updateMany(
await MongoDataset.updateMany(
{
type: { $exists: false }
},
{
$set: {
type: KbTypeEnum.dataset,
type: DatasetTypeEnum.dataset,
parentId: null
}
}
......
projects/app/src/pages/api/admin/initv441.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { authUser } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
import { connectToDatabase } from '@/service/mongo';
import mongoose from 'mongoose';
import mongoose from '@fastgpt/common/mongo';
import { PgClient } from '@/service/pg';
import { PgDatasetTableName } from '@/constants/plugin';
......
projects/app/src/pages/api/admin/initv442.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { authUser } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
import { connectToDatabase, Bill } from '@/service/mongo';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
......
projects/app/src/pages/api/admin/initv445.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { authUser } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
import { connectToDatabase, App } from '@/service/mongo';
import { FlowInputItemTypeEnum, FlowModuleTypeEnum } from '@/constants/flow';
import { SystemInputEnum } from '@/constants/app';
......
projects/app/src/pages/api/admin/initv447.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { authUser } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
import { connectToDatabase } from '@/service/mongo';
import { PgClient } from '@/service/pg';
import { PgDatasetTableName } from '@/constants/plugin';
import { DatasetSpecialIdEnum } from '@fastgpt/core/dataset/constant';
import mongoose, { Types } from 'mongoose';
import { Types, connectionMongo } from '@fastgpt/common/mongo';
import { delay } from '@/utils/tools';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
......@@ -71,7 +71,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
}
async function init(rows: any[], initFileIds: string[]) {
const collection = mongoose.connection.db.collection(`dataset.files`);
const collection = connectionMongo.connection.db.collection(`dataset.files`);
/* 遍历所有的 fileId，去找有没有对应的文件，没有的话则改成manual */
const updateResult = await Promise.allSettled(
......
projects/app/src/pages/api/admin/sqlTest.ts 0 → 100644
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { authUser } from '@fastgpt/support/user/auth';
import { connectToDatabase } from '@/service/mongo';
import { PgClient } from '@/service/pg';
import { PgDatasetTableName } from '@/constants/plugin';
import { DatasetSpecialIdEnum } from '@fastgpt/core/dataset/constant';
import { Types, connectionMongo } from '@fastgpt/common/mongo';
import { delay } from '@/utils/tools';
import { replaceVariable } from '@/utils/common/tools/text';
import { getVector } from '../openapi/plugin/vector';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
// await connectToDatabase();
// const { text, analyze, sql } = req.body as {
// userId: string;
// text: string;
// analyze?: boolean;
// sql: string;
// };
// await authUser({ req, authRoot: true });
// const vectorModel = global.vectorModels[0];
// const { vectors } = await getVector({
// model: vectorModel.model,
// input: [text]
// });
// const start = Date.now();
// const result: any = await PgClient.query(sql.replace(/\[vector\]/g, `[${vectors[0]}]`));
jsonRes(res, {
data: {
// rows: result?.[2]?.rows,
// time: Date.now() - start
}
});
} catch (error) {
jsonRes(res, {
code: 500,
error
});
}
}
projects/app/src/pages/api/app/create.ts
......@@ -2,13 +2,14 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
import { App } from '@/service/models/app';
import type { CreateAppParams } from '@/types/app';
import { AppTypeEnum } from '@/constants/app';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const {
name = 'APP',
avatar,
......@@ -23,8 +24,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
// 上限校验
const authCount = await App.countDocuments({
userId
......
projects/app/src/pages/api/app/data/totalUsage.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, Bill } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { Types } from 'mongoose';
import { authUser } from '@fastgpt/support/user/auth';
import { Types } from '@fastgpt/common/mongo';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { appId, start, end } = req.body as { appId: string; start: number; end: number };
const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
const result = await Bill.aggregate([
{
$match: {
......
projects/app/src/pages/api/app/del.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { Chat, App, connectToDatabase, Collection, OutLink } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { Chat, App, connectToDatabase, Collection } from '@/service/mongo';
import { MongoOutLink } from '@fastgpt/support/outLink/schema';
import { authUser } from '@fastgpt/support/user/auth';
import { authApp } from '@/service/utils/auth';
/* 获取我的模型 */
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { appId } = req.query as { appId: string };
if (!appId) {
......@@ -16,8 +18,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
// 验证是否是该用户的 app
await authApp({
appId,
......@@ -35,7 +35,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
});
// 删除分享链接
await OutLink.deleteMany({
await MongoOutLink.deleteMany({
appId
});
......
projects/app/src/pages/api/app/detail.tsx
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
import { authApp } from '@/service/utils/auth';
/* 获取我的模型 */
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { appId } = req.query as { appId: string };
if (!appId) {
......@@ -16,8 +17,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
const { app } = await authApp({
appId,
userId
......
projects/app/src/pages/api/app/getChatLogs.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { Chat, connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
import type { PagingData } from '@/types';
import { AppLogsListItemType } from '@/types/app';
import { Types } from 'mongoose';
import { Types } from '@fastgpt/common/mongo';
import { addDays } from 'date-fns';
import type { GetAppChatLogsParams } from '@/global/core/api/appReq.d';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const {
pageNum = 1,
pageSize = 20,
......@@ -21,7 +22,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
if (!appId) {
throw new Error('缺少参数');
}
await connectToDatabase();
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
......
projects/app/src/pages/api/app/myApps.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, App } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
import { AppListItemType } from '@/types/app';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
// 根据 userId 获取模型信息
const myApps = await App.find(
{
......
projects/app/src/pages/api/app/share/collection.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, Collection, App } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
/* 模型收藏切换 */
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { appId } = req.query as { appId: string };
if (!appId) {
......@@ -14,8 +15,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
const collectionRecord = await Collection.findOne({
userId,
modelId: appId
......
projects/app/src/pages/api/app/share/getModels.ts
......@@ -3,20 +3,19 @@ import { jsonRes } from '@/service/response';
import { connectToDatabase, App } from '@/service/mongo';
import type { PagingData } from '@/types';
import type { ShareAppItem } from '@/types/app';
import { authUser } from '@/service/utils/auth';
import { Types } from 'mongoose';
import { authUser } from '@fastgpt/support/user/auth';
import { Types } from '@fastgpt/common/mongo';
/* 获取模型列表 */
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const {
searchText = '',
pageNum = 1,
pageSize = 20
} = req.body as { searchText: string; pageNum: number; pageSize: number };
await connectToDatabase();
const { userId } = await authUser({ req, authToken: true });
const regex = new RegExp(searchText, 'i');
......
projects/app/src/pages/api/app/update.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
import { App } from '@/service/models/app';
import type { AppUpdateParams } from '@/types/app';
import { authApp } from '@/service/utils/auth';
......@@ -9,6 +9,7 @@ import { authApp } from '@/service/utils/auth';
/* 获取我的模型 */
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { name, avatar, type, share, intro, modules } = req.body as AppUpdateParams;
const { appId } = req.query as { appId: string };
......@@ -19,8 +20,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
await authApp({
appId,
userId
......
projects/app/src/pages/api/chat/chatTest.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
import { sseErrRes } from '@/service/response';
import { sseResponseEventEnum } from '@/constants/chat';
import { sseResponse } from '@/service/utils/tools';
......@@ -30,6 +30,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
let { modules = [], history = [], prompt, variables = {}, appName, appId } = req.body as Props;
try {
await connectToDatabase();
if (!history || !modules || !prompt) {
throw new Error('Prams Error');
}
......@@ -37,8 +38,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
throw new Error('history is not array');
}
await connectToDatabase();
/* user auth */
const { userId, user } = await authUser({ req, authToken: true, authBalance: true });
......
projects/app/src/pages/api/chat/delChatRecordByContentId.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, ChatItem } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const { chatId, contentId } = req.query as { chatId: string; contentId: string };
await connectToDatabase();
const { chatId, contentId } = req.query as { chatId: string; contentId: string };
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
......
projects/app/src/pages/api/chat/feedback/adminUpdate.ts
......@@ -2,7 +2,7 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, ChatItem } from '@/service/mongo';
import type { AdminUpdateFeedbackParams } from '@/global/core/api/chatReq.d';
import { authUser } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
/* 初始化我的聊天框，需要身份验证 */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
......
projects/app/src/pages/api/chat/history/getHistory.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, Chat } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
import type { ChatHistoryItemType } from '@/types/chat';
import { ChatSourceEnum } from '@/constants/chat';
/* 获取历史记录 */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { appId } = req.body as { appId?: string };
const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
const data = await Chat.find(
{
userId,
......
projects/app/src/pages/api/chat/history/updateChatHistory.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, Chat } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
export type Props = {
chatId: string;
......@@ -12,12 +12,11 @@ export type Props = {
/* 更新聊天标题 */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { chatId, customTitle, top } = req.body as Props;
const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
await Chat.findOneAndUpdate(
{
chatId,
......
projects/app/src/pages/api/chat/init.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { Chat, ChatItem } from '@/service/mongo';
import { Chat, ChatItem, connectToDatabase } from '@/service/mongo';
import type { InitChatResponse } from '@/global/core/api/chatRes.d';
import { authUser } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
import { ChatItemType } from '@/types/chat';
import { authApp } from '@/service/utils/auth';
import type { ChatSchema } from '@/types/mongoSchema';
......@@ -12,6 +12,7 @@ import { TaskResponseKeyEnum } from '@/constants/chat';
/* 初始化我的聊天框，需要身份验证 */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { userId } = await authUser({ req, authToken: true });
let { appId, chatId } = req.query as {
......
projects/app/src/pages/api/chat/removeHistory.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, Chat, ChatItem } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
import { ChatSourceEnum } from '@/constants/chat';
type Props = {
......@@ -12,11 +12,10 @@ type Props = {
/* clear chat history */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { chatId, appId } = req.query as Props;
const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
if (chatId) {
await Promise.all([
Chat.findOneAndRemove({
......
projects/app/src/pages/api/common/bill/createTrainingBill.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, Bill } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
import { BillSourceEnum } from '@/constants/user';
import { CreateTrainingBillType } from '@/global/common/api/billReq.d';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { name } = req.body as CreateTrainingBillType;
const { userId } = await authUser({ req, authToken: true, authApiKey: true });
await connectToDatabase();
const { _id } = await Bill.create({
userId,
appName: name,
......
projects/app/src/pages/api/core/ai/agent/createQuestionGuide.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
import type { CreateQuestionGuideParams } from '@/global/core/api/aiReq.d';
import { pushQuestionGuideBill } from '@/service/common/bill/push';
import { defaultQGModel } from '@/pages/api/system/getInitData';
......
projects/app/src/pages/api/core/dataset/allDataset.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, KB } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { connectToDatabase } from '@/service/mongo';
import { MongoDataset } from '@fastgpt/core/dataset/schema';
import { authUser } from '@fastgpt/support/user/auth';
import { getVectorModel } from '@/service/utils/data';
import type { DatasetsItemType } from '@/types/core/dataset';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
const kbList = await KB.find({
const kbList = await MongoDataset.find({
userId,
type: 'dataset'
});
......
projects/app/src/pages/api/core/dataset/create.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, KB } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { connectToDatabase } from '@/service/mongo';
import { MongoDataset } from '@fastgpt/core/dataset/schema';
import { authUser } from '@fastgpt/support/user/auth';
import type { CreateDatasetParams } from '@/global/core/api/datasetReq.d';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { name, tags, avatar, vectorModel, parentId, type } = req.body as CreateDatasetParams;
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
const { _id } = await KB.create({
const { _id } = await MongoDataset.create({
name,
userId,
tags,
......
projects/app/src/pages/api/core/dataset/data/delDataById.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { authUser } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
import { PgClient } from '@/service/pg';
import { withNextCors } from '@/service/utils/tools';
import { PgDatasetTableName } from '@/constants/plugin';
import { connectToDatabase } from '@/service/mongo';
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
let { dataId } = req.query as {
dataId: string;
};
......
projects/app/src/pages/api/core/dataset/data/exportAll.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, User } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { connectToDatabase } from '@/service/mongo';
import { MongoUser } from '@fastgpt/support/user/schema';
import { authUser } from '@fastgpt/support/user/auth';
import { PgDatasetTableName } from '@/constants/plugin';
import { findAllChildrenIds } from '../delete';
import QueryStream from 'pg-query-stream';
......@@ -11,6 +12,7 @@ import { responseWriteController } from '@/service/common/stream';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
let { kbId } = req.query as {
kbId: string;
};
......@@ -19,8 +21,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
throw new Error('缺少参数');
}
await connectToDatabase();
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
......@@ -31,7 +31,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
);
// auth export times
const authTimes = await User.findOne(
const authTimes = await MongoUser.findOne(
{
_id: userId,
$or: [
......@@ -101,7 +101,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
stream.on('end', async () => {
try {
// update export time
await User.findByIdAndUpdate(userId, {
await MongoUser.findByIdAndUpdate(userId, {
'limit.exportKbTime': new Date()
});
} catch (error) {}
......
projects/app/src/pages/api/core/dataset/data/getDataById.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
import { PgClient } from '@/service/pg';
import { PgDatasetTableName } from '@/constants/plugin';
import type { PgDataItemType } from '@/types/core/dataset/data';
......@@ -15,6 +15,7 @@ export type Response = {
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
let { dataId } = req.query as {
dataId: string;
};
......@@ -25,8 +26,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
const where: any = [['user_id', userId], 'AND', ['id', dataId]];
const searchRes = await PgClient.select<PgDataItemType>(PgDatasetTableName, {
......
projects/app/src/pages/api/core/dataset/data/getDataList.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
import { PgClient } from '@/service/pg';
import { PgDatasetTableName } from '@/constants/plugin';
import type { PgDataItemType } from '@/types/core/dataset/data';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
let {
kbId,
pageNum = 1,
......@@ -28,7 +29,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
searchText = searchText.replace(/'/g, '');
const where: any = [
......
projects/app/src/pages/api/core/dataset/data/getQueueLen.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { TrainingData } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { TrainingData, connectToDatabase } from '@/service/mongo';
import { authUser } from '@fastgpt/support/user/auth';
/* 拆分数据成QA */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
await authUser({ req, authToken: true });
// split queue data
......
projects/app/src/pages/api/core/dataset/data/getTrainingData.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, TrainingData } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
import { TrainingModeEnum } from '@/constants/plugin';
import { Types } from 'mongoose';
import { Types } from '@fastgpt/common/mongo';
import { startQueue } from '@/service/utils/tools';
/* 拆分数据成QA */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { kbId, init = false } = req.body as { kbId: string; init: boolean };
if (!kbId) {
throw new Error('参数错误');
}
await connectToDatabase();
const { userId } = await authUser({ req, authToken: true });
......
projects/app/src/pages/api/core/dataset/data/insertData.ts
......@@ -5,7 +5,8 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authKb, authUser } from '@/service/utils/auth';
import { authDataset } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
import { withNextCors } from '@/service/utils/tools';
import { PgDatasetTableName } from '@/constants/plugin';
import { insertData2Dataset, PgClient } from '@/service/pg';
......@@ -50,7 +51,7 @@ export async function getVectorAndInsertDataset(
}
// auth kb
const kb = await authKb({ kbId, userId });
const kb = await authDataset({ kbId, userId });
const q = data?.q?.replace(/\\n/g, '\n').trim().replace(/'/g, '"');
const a = data?.a?.replace(/\\n/g, '\n').trim().replace(/'/g, '"');
......
projects/app/src/pages/api/core/dataset/data/pushData.ts
/* push data to training queue */
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, TrainingData, KB } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { authKb } from '@/service/utils/auth';
import { connectToDatabase, TrainingData } from '@/service/mongo';
import { MongoDataset } from '@fastgpt/core/dataset/schema';
import { authUser } from '@fastgpt/support/user/auth';
import { authDataset } from '@/service/utils/auth';
import { withNextCors } from '@/service/utils/tools';
import { TrainingModeEnum } from '@/constants/plugin';
import { startQueue } from '@/service/utils/tools';
......@@ -21,6 +22,7 @@ const modeMap = {
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { kbId, data, mode = TrainingModeEnum.index } = req.body as PushDataProps;
if (!kbId || !Array.isArray(data)) {
......@@ -35,8 +37,6 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
throw new Error('Data is too long, max 500');
}
await connectToDatabase();
// 凭证校验
const { userId } = await authUser({ req, authToken: true, authApiKey: true });
......@@ -63,13 +63,13 @@ export async function pushDataToKb({
billId
}: { userId: string } & PushDataProps): Promise<PushDataResponse> {
const [kb, vectorModel] = await Promise.all([
authKb({
authDataset({
userId,
kbId
}),
(async () => {
if (mode === TrainingModeEnum.index) {
const vectorModel = (await KB.findById(kbId, 'vectorModel'))?.vectorModel;
const vectorModel = (await MongoDataset.findById(kbId, 'vectorModel'))?.vectorModel;
return getVectorModel(vectorModel || global.vectorModels[0].model);
}
......
projects/app/src/pages/api/core/dataset/data/updateData.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { authUser } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
import { PgClient } from '@/service/pg';
import { withNextCors } from '@/service/utils/tools';
import { KB, connectToDatabase } from '@/service/mongo';
import { connectToDatabase } from '@/service/mongo';
import { MongoDataset } from '@fastgpt/core/dataset/schema';
import { getVector } from '@/pages/api/openapi/plugin/vector';
import { PgDatasetTableName } from '@/constants/plugin';
import type { UpdateDatasetDataPrams } from '@/global/core/api/datasetReq.d';
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { dataId, a = '', q = '', kbId } = req.body as UpdateDatasetDataPrams;
if (!dataId) {
throw new Error('缺少参数');
}
await connectToDatabase();
// auth user and get kb
const [{ userId }, kb] = await Promise.all([
authUser({ req, authToken: true }),
KB.findById(kbId, 'vectorModel')
MongoDataset.findById(kbId, 'vectorModel')
]);
if (!kb) {
......
projects/app/src/pages/api/core/dataset/delete.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, KB, TrainingData } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { connectToDatabase, TrainingData } from '@/service/mongo';
import { MongoDataset } from '@fastgpt/core/dataset/schema';
import { authUser } from '@fastgpt/support/user/auth';
import { PgClient } from '@/service/pg';
import { PgDatasetTableName } from '@/constants/plugin';
import { GridFSStorage } from '@/service/lib/gridfs';
import { Types } from 'mongoose';
import { Types } from '@fastgpt/common/mongo';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
......@@ -43,7 +44,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
await Promise.all(deletedIds.map((id) => gridFs.deleteFilesByKbId(id)));
// delete kb data
await KB.deleteMany({
await MongoDataset.deleteMany({
_id: { $in: deletedIds },
userId
});
......@@ -59,7 +60,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
export async function findAllChildrenIds(id: string) {
// find children
const children = await KB.find({ parentId: id });
const children = await MongoDataset.find({ parentId: id });
let allChildrenIds = children.map((child) => String(child._id));
......
projects/app/src/pages/api/core/dataset/detail.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, KB } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@fastgpt/support/user/auth';
import { getVectorModel } from '@/service/utils/data';
import { MongoDataset } from '@fastgpt/core/dataset/schema';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { id } = req.query as {
id: string;
};
......@@ -17,9 +19,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
const data = await KB.findOne({
const data = await MongoDataset.findOne({
_id: id,
userId
});
......
projects/app/src/pages/api/core/dataset/file/delById.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, TrainingData } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
import { GridFSStorage } from '@/service/lib/gridfs';
import { PgClient } from '@/service/pg';
import { PgDatasetTableName } from '@/constants/plugin';
import { Types } from 'mongoose';
import { Types } from '@fastgpt/common/mongo';
import { isSpecialFileId } from '@fastgpt/core/dataset/utils';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
......
projects/app/src/pages/api/core/dataset/file/delEmptyFiles.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
import { GridFSStorage } from '@/service/lib/gridfs';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
......
projects/app/src/pages/api/core/dataset/file/detail.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
import { GridFSStorage } from '@/service/lib/gridfs';
import { datasetSpecialIdMap } from '@fastgpt/core/dataset/constant';
import { datasetSpecialIds } from '@fastgpt/core/dataset/constant';
......
projects/app/src/pages/api/core/dataset/file/list.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, TrainingData } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
import { GridFSStorage } from '@/service/lib/gridfs';
import { PgClient, updateDataFileId } from '@/service/pg';
import { PgDatasetTableName } from '@/constants/plugin';
import { FileStatusEnum } from '@/constants/dataset';
import { strIsLink } from '@fastgpt/common/tools/str';
import {
FileStatusEnum,
DatasetSpecialIdEnum,
datasetSpecialIdMap,
datasetSpecialIds
} from '@fastgpt/core/dataset/constant';
import { Types } from 'mongoose';
import { Types } from '@fastgpt/common/mongo';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
......
projects/app/src/pages/api/core/dataset/file/markUsed.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
import { GridFSStorage } from '@/service/lib/gridfs';
import { MarkFileUsedProps } from '@/global/core/api/datasetReq.d';
import { Types } from 'mongoose';
import { Types } from '@fastgpt/common/mongo';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
......
projects/app/src/pages/api/core/dataset/file/update.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
import { GridFSStorage } from '@/service/lib/gridfs';
import { UpdateFileProps } from '@/global/core/api/datasetReq.d';
import { Types } from 'mongoose';
import { Types } from '@fastgpt/common/mongo';
import { PgClient } from '@/service/pg';
import { PgDatasetTableName } from '@/constants/plugin';
import { addLog } from '@/service/utils/tools';
......
projects/app/src/pages/api/core/dataset/list.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, KB } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@fastgpt/support/user/auth';
import { getVectorModel } from '@/service/utils/data';
import type { DatasetsItemType } from '@/types/core/dataset';
import { KbTypeEnum } from '@/constants/dataset';
import { DatasetTypeEnum } from '@fastgpt/core/dataset/constant';
import { MongoDataset } from '@fastgpt/core/dataset/schema';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
const { parentId, type } = req.query as { parentId?: string; type?: `${KbTypeEnum}` };
await connectToDatabase();
const { parentId, type } = req.query as { parentId?: string; type?: `${DatasetTypeEnum}` };
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
const kbList = await KB.find({
const kbList = await MongoDataset.find({
userId,
...(parentId !== undefined && { parentId: parentId || null }),
...(type && { type })
......
projects/app/src/pages/api/core/dataset/paths.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, KB } from '@/service/mongo';
import { connectToDatabase } from '@/service/mongo';
import { MongoDataset } from '@fastgpt/core/dataset/schema';
import type { DatasetPathItemType } from '@/types/core/dataset';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
......@@ -25,7 +26,7 @@ async function getParents(parentId?: string): Promise<DatasetPathItemType[]> {
return [];
}
const parent = await KB.findById(parentId, 'name parentId');
const parent = await MongoDataset.findById(parentId, 'name parentId');
if (!parent) return [];
......
projects/app/src/pages/api/core/dataset/searchTest.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { authUser } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
import { PgClient } from '@/service/pg';
import { withNextCors } from '@/service/utils/tools';
import { getVector } from '../../openapi/plugin/vector';
import { PgDatasetTableName } from '@/constants/plugin';
import { KB } from '@/service/mongo';
import { MongoDataset } from '@fastgpt/core/dataset/schema';
import type { SearchTestProps } from '@/global/core/api/datasetReq.d';
import type { SearchTestResponseType } from '@/global/core/api/datasetRes.d';
import { connectToDatabase } from '@/service/mongo';
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { kbId, text } = req.body as SearchTestProps;
if (!kbId || !text) {
......@@ -20,7 +22,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
// 凭证校验
const [{ userId }, kb] = await Promise.all([
authUser({ req, authToken: true, authApiKey: true }),
KB.findById(kbId, 'vectorModel')
MongoDataset.findById(kbId, 'vectorModel')
]);
if (!userId || !kb) {
......@@ -35,7 +37,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
const response: any = await PgClient.query(
`BEGIN;
SET LOCAL ivfflat.probes = ${global.systemEnv.pgIvfflatProbe || 10};
SET LOCAL hnsw.ef_search= ${global.systemEnv.pgHNSWEfSearch || 40};
select id, q, a, source, file_id, (vector <#> '[${
vectors[0]
}]') * -1 AS score from ${PgDatasetTableName} where kb_id='${kbId}' AND user_id='${userId}' order by vector <#> '[${
......
projects/app/src/pages/api/core/dataset/update.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, KB } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { connectToDatabase } from '@/service/mongo';
import { MongoDataset } from '@fastgpt/core/dataset/schema';
import { authUser } from '@fastgpt/support/user/auth';
import type { DatasetUpdateParams } from '@/global/core/api/datasetReq.d';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { id, parentId, name, avatar, tags } = req.body as DatasetUpdateParams;
if (!id) {
......@@ -15,9 +17,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
await KB.findOneAndUpdate(
await MongoDataset.findOneAndUpdate(
{
_id: id,
userId
......
projects/app/src/pages/api/openapi/plugin/vector.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { authBalanceByUid, authUser } from '@/service/utils/auth';
import { authBalanceByUid, authUser } from '@fastgpt/support/user/auth';
import { withNextCors } from '@/service/utils/tools';
import { getAIApi } from '@fastgpt/core/ai/config';
import { pushGenerateVectorBill } from '@/service/common/bill/push';
import { connectToDatabase } from '@/service/mongo';
type Props = {
model: string;
......@@ -17,6 +18,7 @@ type Response = {
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { userId } = await authUser({ req, authToken: true });
let { input, model } = req.query as Props;
......
projects/app/src/pages/api/plugins/urlFetch.ts
......@@ -4,14 +4,16 @@ import axios from 'axios';
import { JSDOM } from 'jsdom';
import { Readability } from '@mozilla/readability';
import { jsonRes } from '@/service/response';
import { authUser } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
import type { FetchResultItem } from '@/global/common/api/pluginRes.d';
import { simpleText } from '@/utils/file';
import { simpleText } from '@fastgpt/common/tools/str';
import { connectToDatabase } from '@/service/mongo';
export type UrlFetchResponse = FetchResultItem[];
const fetchContent = async (req: NextApiRequest, res: NextApiResponse) => {
try {
await connectToDatabase();
let { urlList = [] } = req.body as { urlList: string[] };
if (!urlList || urlList.length === 0) {
......
projects/app/src/pages/api/plusApi/[...path].ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { request } from '@/service/common/api/request';
import { request } from '@fastgpt/common/plusApi/request';
import type { Method } from 'axios';
import { connectToDatabase } from '@/service/mongo';
......
projects/app/src/pages/api/support/openapi/delKey.ts
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, OpenApi } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { connectToDatabase } from '@/service/mongo';
import { MongoOpenApi } from '@fastgpt/support/openapi/schema';
import { authUser } from '@fastgpt/support/user/auth';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { id } = req.query as { id: string };
if (!id) {
......@@ -14,9 +16,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
await OpenApi.findOneAndRemove({ _id: id, userId });
await MongoOpenApi.findOneAndRemove({ _id: id, userId });
jsonRes(res);
} catch (err) {
......
projects/app/src/pages/api/support/openapi/getKeys.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, OpenApi } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { connectToDatabase } from '@/service/mongo';
import { MongoOpenApi } from '@fastgpt/support/openapi/schema';
import { authUser } from '@fastgpt/support/user/auth';
import type { GetApiKeyProps } from '@/global/support/api/openapiReq.d';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { appId } = req.query as GetApiKeyProps;
const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
const findResponse = await OpenApi.find({ userId, appId }).sort({ _id: -1 });
const findResponse = await MongoOpenApi.find({ userId, appId }).sort({ _id: -1 });
jsonRes(res, {
data: findResponse.map((item) => item.toObject())
......
projects/app/src/pages/api/support/openapi/postKey.ts
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, OpenApi } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { connectToDatabase } from '@/service/mongo';
import { MongoOpenApi } from '@fastgpt/support/openapi/schema';
import { authUser } from '@fastgpt/support/user/auth';
import { customAlphabet } from 'nanoid';
import type { EditApiKeyProps } from '@/global/support/api/openapiReq.d';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { appId, name, limit } = req.body as EditApiKeyProps;
const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
const count = await OpenApi.find({ userId, appId }).countDocuments();
const count = await MongoOpenApi.find({ userId, appId }).countDocuments();
if (count >= 10) {
throw new Error('最多 10 组 API 秘钥');
......@@ -25,7 +25,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
);
const apiKey = `${global.systemEnv?.openapiPrefix || 'fastgpt'}-${nanoid()}`;
await OpenApi.create({
await MongoOpenApi.create({
userId,
apiKey,
appId,
......
projects/app/src/pages/api/support/openapi/putKey.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, OpenApi } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { connectToDatabase } from '@/service/mongo';
import { MongoOpenApi } from '@fastgpt/support/openapi/schema';
import { authUser } from '@fastgpt/support/user/auth';
import type { EditApiKeyProps } from '@/global/support/api/openapiReq.d';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { _id, name, limit } = req.body as EditApiKeyProps & { _id: string };
const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
await OpenApi.findOneAndUpdate(
await MongoOpenApi.findOneAndUpdate(
{
_id,
userId
......
projects/app/src/pages/api/support/outLink/create.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, OutLink } from '@/service/mongo';
import { authApp, authUser } from '@/service/utils/auth';
import type { OutLinkEditType } from '@/types/support/outLink';
import { connectToDatabase } from '@/service/mongo';
import { MongoOutLink } from '@fastgpt/support/outLink/schema';
import { authApp } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
import type { OutLinkEditType } from '@fastgpt/support/outLink/type.d';
import { customAlphabet } from 'nanoid';
import { OutLinkTypeEnum } from '@/constants/chat';
import { OutLinkTypeEnum } from '@fastgpt/support/outLink/constant';
const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 24);
/* create a shareChat */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { appId, ...props } = req.body as OutLinkEditType & {
appId: string;
type: `${OutLinkTypeEnum}`;
};
await connectToDatabase();
const { userId } = await authUser({ req, authToken: true });
await authApp({
appId,
......@@ -25,7 +26,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
});
const shareId = nanoid();
await OutLink.create({
await MongoOutLink.create({
shareId,
userId,
appId,
......
projects/app/src/pages/api/support/outLink/delete.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, OutLink } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { connectToDatabase } from '@/service/mongo';
import { MongoOutLink } from '@fastgpt/support/outLink/schema';
import { authUser } from '@fastgpt/support/user/auth';
/* delete a shareChat by shareChatId */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
......@@ -14,7 +15,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
const { userId } = await authUser({ req, authToken: true });
await OutLink.findOneAndRemove({
await MongoOutLink.findOneAndRemove({
_id: id,
userId
});
......
projects/app/src/pages/api/support/outLink/init.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, OutLink, User } from '@/service/mongo';
import { connectToDatabase } from '@/service/mongo';
import { MongoOutLink } from '@fastgpt/support/outLink/schema';
import { MongoUser } from '@fastgpt/support/user/schema';
import type { InitShareChatResponse } from '@/global/support/api/outLinkRes.d';
import { authApp } from '@/service/utils/auth';
import { HUMAN_ICON } from '@/constants/chat';
import { getChatModelNameList, getGuideModule } from '@/components/ChatBox/utils';
import { authShareChatInit } from '@/service/support/outLink/auth';
import { authShareChatInit } from '@fastgpt/support/outLink/auth';
/* init share chat window */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
let { shareId, authToken } = req.query as {
shareId: string;
authToken?: string;
......@@ -19,10 +22,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
throw new Error('params is error');
}
await connectToDatabase();
// get shareChat
const shareChat = await OutLink.findOne({ shareId });
const shareChat = await MongoOutLink.findOne({ shareId });
if (!shareChat) {
return jsonRes(res, {
......@@ -38,8 +39,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
userId: String(shareChat.userId),
authOwner: false
}),
User.findById(shareChat.userId, 'avatar'),
authShareChatInit(authToken, shareChat.limit?.hookUrl)
MongoUser.findById(shareChat.userId, 'avatar'),
authShareChatInit({
authToken,
tokenUrl: shareChat.limit?.hookUrl
})
]);
jsonRes<InitShareChatResponse>(res, {
......
projects/app/src/pages/api/support/outLink/list.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, OutLink } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { hashPassword } from '@/service/utils/tools';
import { connectToDatabase } from '@/service/mongo';
import { MongoOutLink } from '@fastgpt/support/outLink/schema';
import { authUser } from '@fastgpt/support/user/auth';
/* get shareChat list by appId */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
......@@ -15,7 +15,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
const { userId } = await authUser({ req, authToken: true });
const data = await OutLink.find({
const data = await MongoOutLink.find({
appId,
userId
}).sort({
......
projects/app/src/pages/api/support/outLink/update.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, OutLink } from '@/service/mongo';
import type { OutLinkEditType } from '@/types/support/outLink';
import { connectToDatabase } from '@/service/mongo';
import { MongoOutLink } from '@fastgpt/support/outLink/schema';
import type { OutLinkEditType } from '@fastgpt/support/outLink/type.d';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
......@@ -9,7 +10,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
const { _id, name, responseDetail, limit } = req.body as OutLinkEditType & {};
await OutLink.findByIdAndUpdate(_id, {
await MongoOutLink.findByIdAndUpdate(_id, {
name,
responseDetail,
limit
......
projects/app/src/pages/api/system/file/delete.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
import { GridFSStorage } from '@/service/lib/gridfs';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
......
projects/app/src/pages/api/system/file/readUrl.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
import jwt from 'jsonwebtoken';
import { ERROR_ENUM } from '@/service/errorCode';
import { ERROR_ENUM } from '@fastgpt/common/constant/errorCode';
import { GridFSStorage } from '@/service/lib/gridfs';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
......
projects/app/src/pages/api/system/file/upload.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
import { GridFSStorage } from '@/service/lib/gridfs';
import { customAlphabet } from 'nanoid';
import multer from 'multer';
......
projects/app/src/pages/api/system/file/uploadImage.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, Image } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
type Props = { base64Img: string };
......
projects/app/src/pages/api/system/getInitData.ts
import type { FeConfigsType, SystemEnvType } from '@/types';
import type { SystemEnvType } from '@/types';
import type { FeConfigsType } from '@fastgpt/common/type/index.d';
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { readFileSync } from 'fs';
import type { InitDateResponse } from '@/global/common/api/systemRes';
import { type VectorModelItemType, FunctionModelItemType } from '@/types/model';
import type { VectorModelItemType, FunctionModelItemType } from '@/types/model';
import { formatPrice } from '@fastgpt/common/bill';
import { getTikTokenEnc } from '@/utils/common/tiktoken';
import { initHttpAgent } from '@fastgpt/core/init';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
if (!global.feConfigs) {
await getInitConfig();
}
getInitConfig();
getModelPrice();
jsonRes<InitDateResponse>(res, {
data: {
feConfigs: global.feConfigs,
chatModels: global.chatModels,
qaModel: global.qaModel,
vectorModels: global.vectorModels,
priceMd: global.priceMd,
systemVersion: global.systemVersion || '0.0.0'
}
});
......@@ -23,7 +28,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
const defaultSystemEnv: SystemEnvType = {
vectorMaxProcess: 15,
qaMaxProcess: 15,
pgIvfflatProbe: 20
pgHNSWEfSearch: 40
};
const defaultFeConfigs: FeConfigsType = {
show_emptyChat: true,
......@@ -104,7 +109,17 @@ const defaultVectorModels: VectorModelItemType[] = [
}
];
export async function getInitConfig() {
export function initGlobal() {
// init tikToken
getTikTokenEnc();
initHttpAgent();
global.qaQueueLen = 0;
global.vectorQueueLen = 0;
global.sendInformQueue = [];
global.sendInformQueueLen = 0;
}
export function getInitConfig() {
try {
if (global.feConfigs) return;
......@@ -143,6 +158,7 @@ export function setDefaultData() {
global.extractModel = defaultExtractModel;
global.cqModel = defaultCQModel;
global.qgModel = defaultQGModel;
global.priceMd = '';
}
export function getSystemVersion() {
......@@ -160,3 +176,21 @@ export function getSystemVersion() {
global.systemVersion = '0.0.0';
}
}
function getModelPrice() {
if (global.priceMd) return;
global.priceMd = `| 计费项 | 价格: 元/ 1K tokens(包含上下文)|
| --- | --- |
${global.vectorModels
?.map((item) => `| 索引-${item.name} | ${formatPrice(item.price, 1000)} |`)
.join('\n')}
${global.chatModels
?.map((item) => `| 对话-${item.name} | ${formatPrice(item.price, 1000)} |`)
.join('\n')}
| 文件QA拆分 | ${formatPrice(global.qaModel?.price, 1000)} |
| 高级编排 - 问题分类 | ${formatPrice(global.cqModel?.price, 1000)} |
| 高级编排 - 内容提取 | ${formatPrice(global.extractModel?.price, 1000)} |
| 下一步指引 | ${formatPrice(global.qgModel?.price, 1000)} |
`;
console.log(global.priceMd);
}
projects/app/src/pages/api/user/account/loginByPassword.ts
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { MongoUser } from '@fastgpt/support/user/schema';
import { setCookie } from '@/service/utils/tools';
import { generateToken } from '@fastgpt/support/user/tools';
import { connectToDatabase } from '@/service/mongo';
import { User } from '@/service/models/user';
import { generateToken, setCookie } from '@/service/utils/tools';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { username, password } = req.body;
if (!username || !password) {
throw new Error('缺少参数');
}
await connectToDatabase();
// 检测用户是否存在
const authUser = await User.findOne({
const authUser = await MongoUser.findOne({
username
});
if (!authUser) {
throw new Error('用户未注册');
}
const user = await User.findOne({
const user = await MongoUser.findOne({
username,
password
});
......
projects/app/src/pages/api/user/account/paySuccess.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { TrainingData } from '@/service/mongo';
import { TrainingData, connectToDatabase } from '@/service/mongo';
import { startQueue } from '@/service/utils/tools';
import { authUser } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { userId } = await authUser({ req, authToken: true });
await unlockTask(userId);
} catch (error) {}
......
projects/app/src/pages/api/user/account/tokenLogin.ts
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { authUser } from '@fastgpt/support/user/auth';
import { MongoUser } from '@fastgpt/support/user/schema';
import { connectToDatabase } from '@/service/mongo';
import { User } from '@/service/models/user';
import { authUser } from '@/service/utils/auth';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
const { userId } = await authUser({ req, authToken: true });
// 根据 id 获取用户信息
const user = await User.findById(userId);
const user = await MongoUser.findById(userId);
if (!user) {
throw new Error('账号异常');
......
projects/app/src/pages/api/user/account/update.ts
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { User } from '@/service/models/user';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { MongoUser } from '@fastgpt/support/user/schema';
import { authUser } from '@fastgpt/support/user/auth';
import { UserUpdateParams } from '@/types/user';
import { getAIApi, openaiBaseUrl } from '@fastgpt/core/ai/config';
import { connectToDatabase } from '@/service/mongo';
/* update user info */
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { avatar, timezone, openaiAccount } = req.body as UserUpdateParams;
const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
// auth key
if (openaiAccount?.key) {
console.log('auth user openai key', openaiAccount?.key);
......@@ -35,7 +34,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
}
// 更新对应的记录
await User.updateOne(
await MongoUser.updateOne(
{
_id: userId
},
......
projects/app/src/pages/api/user/account/updatePasswordByOld.ts
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { User } from '@/service/models/user';
import { authUser } from '@fastgpt/support/user/auth';
import { MongoUser } from '@fastgpt/support/user/schema';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { oldPsw, newPsw } = req.body as { oldPsw: string; newPsw: string };
if (!oldPsw || !newPsw) {
throw new Error('Params is missing');
}
await connectToDatabase();
const { userId } = await authUser({ req, authToken: true });
// auth old password
const user = await User.findOne({
const user = await MongoUser.findOne({
_id: userId,
password: oldPsw
});
......@@ -28,7 +27,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
}
// 更新对应的记录
await User.findByIdAndUpdate(userId, {
await MongoUser.findByIdAndUpdate(userId, {
password: newPsw
});
......
projects/app/src/pages/api/user/getBill.ts
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, Bill } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { Bill, connectToDatabase } from '@/service/mongo';
import { authUser } from '@fastgpt/support/user/auth';
import { adaptBill } from '@/utils/adapt';
import { addDays } from 'date-fns';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const {
pageNum = 1,
pageSize = 10,
......@@ -22,8 +23,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
const where = {
userId,
time: {
......
projects/app/src/pages/api/user/getPayOrders.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { authUser } from '@/service/utils/auth';
import { connectToDatabase, Pay } from '@/service/mongo';
import { authUser } from '@fastgpt/support/user/auth';
import { Pay, connectToDatabase } from '@/service/mongo';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
const { userId } = await authUser({ req, authToken: true });
const records = await Pay.find({
userId,
......
projects/app/src/pages/api/user/inform/countUnread.ts
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, Inform } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { Inform, connectToDatabase } from '@/service/mongo';
import { authUser } from '@fastgpt/support/user/auth';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
if (!req.headers.cookie) {
return jsonRes(res, {
data: 0
......@@ -13,8 +14,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
}
const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
const data = await Inform.countDocuments({
userId,
read: false
......
projects/app/src/pages/api/user/inform/list.ts
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, Inform } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { Inform, connectToDatabase } from '@/service/mongo';
import { authUser } from '@fastgpt/support/user/auth';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { userId } = await authUser({ req, authToken: true });
const { pageNum, pageSize = 10 } = req.body as {
......@@ -13,8 +14,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
pageSize: number;
};
await connectToDatabase();
const [informs, total] = await Promise.all([
Inform.find({ userId })
.sort({ time: -1 }) // 按照创建时间倒序排列
......
projects/app/src/pages/api/user/inform/read.ts
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, Inform } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { Inform, connectToDatabase } from '@/service/mongo';
import { authUser } from '@fastgpt/support/user/auth';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
const { userId } = await authUser({ req, authToken: true });
const { id } = req.query as { id: string };
......
projects/app/src/pages/api/user/inform/send.ts
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, Inform, User } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { Inform, connectToDatabase } from '@/service/mongo';
import { authUser } from '@fastgpt/support/user/auth';
import { InformTypeEnum } from '@/constants/user';
import { startSendInform } from '@/service/events/sendInform';
import { MongoUser } from '@fastgpt/support/user/schema';
export type Props = {
type: `${InformTypeEnum}`;
......@@ -15,9 +16,8 @@ export type Props = {
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await authUser({ req, authRoot: true });
await connectToDatabase();
await authUser({ req, authRoot: true });
jsonRes(res, {
data: await sendInform(req.body),
......@@ -62,7 +62,7 @@ export async function sendInform({ type, title, content, userId }: Props) {
}
// send to all user
const users = await User.find({}, '_id');
const users = await MongoUser.find({}, '_id');
await Inform.insertMany(
users.map(({ _id }) => ({
type,
......
projects/app/src/pages/api/user/promotion/getPromotionData.ts
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, User, promotionRecord } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import mongoose from 'mongoose';
import { connectToDatabase, promotionRecord } from '@/service/mongo';
import { authUser } from '@fastgpt/support/user/auth';
import mongoose from '@fastgpt/common/mongo';
import { MongoUser } from '@fastgpt/support/user/schema';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { userId } = await authUser({ req, authToken: true });
const invitedAmount = await User.countDocuments({
const invitedAmount = await MongoUser.countDocuments({
inviterId: userId
});
......
projects/app/src/pages/api/user/promotion/getPromotions.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, promotionRecord } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
let { pageNum = 1, pageSize = 10 } = req.body as {
pageNum: number;
pageSize: number;
......@@ -12,8 +13,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
const data = await promotionRecord
.find(
{
......
projects/app/src/pages/api/v1/chat/completions.ts
import type { NextApiRequest, NextApiResponse } from 'next';
import { connectToDatabase } from '@/service/mongo';
import { authUser, authApp, AuthUserTypeEnum } from '@/service/utils/auth';
import { authApp } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
import { AuthUserTypeEnum } from '@fastgpt/support/user/auth';
import { sseErrRes, jsonRes } from '@/service/response';
import { addLog, withNextCors } from '@/service/utils/tools';
import { ChatRoleEnum, ChatSourceEnum, sseResponseEventEnum } from '@/constants/chat';
......@@ -27,16 +28,17 @@ import { AppModuleItemType, RunningModuleItemType } from '@/types/app';
import { pushChatBill } from '@/service/common/bill/push';
import { BillSourceEnum } from '@/constants/user';
import { ChatHistoryItemResType } from '@/types/chat';
import { UserModelSchema } from '@/types/mongoSchema';
import type { UserModelSchema } from '@fastgpt/support/user/type.d';
import { SystemInputEnum } from '@/constants/app';
import { getSystemTime } from '@/utils/user';
import { authOutLinkChat } from '@/service/support/outLink/auth';
import { authOutLinkChat } from '@fastgpt/support/outLink/auth';
import { pushResult2Remote, updateOutLinkUsage } from '@fastgpt/support/outLink/tools';
import requestIp from 'request-ip';
import { replaceVariable } from '@/utils/common/tools/text';
import type { ModuleDispatchProps } from '@/types/core/chat/type';
import { selectShareResponse } from '@/utils/service/core/chat';
import { pushResult2Remote, updateOutLinkUsage } from '@/service/support/outLink';
import { updateApiKeyUsage } from '@/service/support/openapi';
import { updateApiKeyUsage } from '@fastgpt/support/openapi/tools';
import { connectToDatabase } from '@/service/mongo';
type FastGptWebChatProps = {
chatId?: string; // undefined: nonuse history, '': new chat, 'xxxxx': use history
......@@ -80,6 +82,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
} = req.body as Props;
try {
await connectToDatabase();
// body data check
if (!messages) {
throw new Error('Prams Error');
......@@ -91,7 +94,6 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
throw new Error('messages is empty');
}
await connectToDatabase();
let startTime = Date.now();
/* user auth */
......
projects/app/src/pages/api/v1/chat/getHistory.ts
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { authUser } from '@/service/utils/auth';
import { connectToDatabase, ChatItem } from '@/service/mongo';
import { Types } from 'mongoose';
import { authUser } from '@fastgpt/support/user/auth';
import { ChatItem, connectToDatabase } from '@/service/mongo';
import { Types } from '@fastgpt/common/mongo';
import type { ChatItemType } from '@/types/chat';
export type Props = {
......
projects/app/src/pages/app/detail/components/DatasetSelectModal.tsx
......@@ -20,7 +20,7 @@ import MySlider from '@/components/Slider';
import MyTooltip from '@/components/MyTooltip';
import MyModal from '@/components/MyModal';
import MyIcon from '@/components/Icon';
import { KbTypeEnum } from '@/constants/dataset';
import { DatasetTypeEnum } from '@fastgpt/core/dataset/constant';
import { useTranslation } from 'react-i18next';
import { useQuery } from '@tanstack/react-query';
import { useDatasetStore } from '@/web/core/store/dataset';
......@@ -116,7 +116,7 @@ export const DatasetSelectModal = ({
<MyTooltip
key={item._id}
label={
item.type === KbTypeEnum.dataset
item.type === DatasetTypeEnum.dataset
? t('kb.Select Dataset')
: t('kb.Select Folder')
}
......@@ -131,9 +131,9 @@ export const DatasetSelectModal = ({
boxShadow: 'md'
}}
onClick={() => {
if (item.type === KbTypeEnum.folder) {
if (item.type === DatasetTypeEnum.folder) {
setParentId(item._id);
} else if (item.type === KbTypeEnum.dataset) {
} else if (item.type === DatasetTypeEnum.dataset) {
const vectorModel = selectedKbList[0]?.vectorModel?.model;
if (vectorModel && vectorModel !== item.vectorModel.model) {
......@@ -161,7 +161,7 @@ export const DatasetSelectModal = ({
</Box>
</Flex>
<Flex justifyContent={'flex-end'} alignItems={'center'} fontSize={'sm'}>
{item.type === KbTypeEnum.folder ? (
{item.type === DatasetTypeEnum.folder ? (
<Box color={'myGray.500'}>{t('Folder')}</Box>
) : (
<>
......
projects/app/src/pages/app/detail/components/OutLink/Share.tsx
......@@ -34,10 +34,10 @@ import { formatTimeToChatTime } from '@/utils/tools';
import { useCopyData } from '@/web/common/hooks/useCopyData';
import { useForm } from 'react-hook-form';
import { defaultOutLinkForm } from '@/constants/model';
import type { OutLinkEditType } from '@/types/support/outLink';
import type { OutLinkEditType } from '@fastgpt/support/outLink/type.d';
import { useRequest } from '@/web/common/hooks/useRequest';
import { formatPrice } from '@fastgpt/common/bill/index';
import { OutLinkTypeEnum } from '@/constants/chat';
import { OutLinkTypeEnum } from '@fastgpt/support/outLink/constant';
import { useTranslation } from 'react-i18next';
import { useToast } from '@/web/common/hooks/useToast';
import { feConfigs } from '@/web/common/store/static';
......
projects/app/src/pages/app/detail/components/OutLink/index.tsx
import React, { useState } from 'react';
import { Box, useTheme } from '@chakra-ui/react';
import { OutLinkTypeEnum } from '@/constants/chat';
import { OutLinkTypeEnum } from '@fastgpt/support/outLink/constant';
import dynamic from 'next/dynamic';
import MyRadio from '@/components/Radio';
......
projects/app/src/pages/components/Navbar.tsx
......@@ -23,6 +23,16 @@ const Navbar = () => {
...(feConfigs?.show_contact
? [
{
label: t('home.Commercial'),
key: 'community',
onClick: () => {
window.open(
'https://fael3z0zfze.feishu.cn/share/base/form/shrcnRxj3utrzjywsom96Px4sud',
'_blank'
);
}
},
{
label: t('home.Community'),
key: 'community',
onClick: () => {
......
projects/app/src/pages/kb/detail/components/FileCard.tsx
......@@ -28,14 +28,13 @@ import MyInput from '@/components/MyInput';
import dayjs from 'dayjs';
import { useRequest } from '@/web/common/hooks/useRequest';
import { useLoading } from '@/web/common/hooks/useLoading';
import { FileStatusEnum } from '@/constants/dataset';
import { useRouter } from 'next/router';
import { usePagination } from '@/web/common/hooks/usePagination';
import type { DatasetFileItemType } from '@/types/core/dataset/file';
import { useGlobalStore } from '@/web/common/store/global';
import MyMenu from '@/components/MyMenu';
import { useEditTitle } from '@/web/common/hooks/useEditTitle';
import { datasetSpecialIds } from '@fastgpt/core/dataset/constant';
import { datasetSpecialIds, FileStatusEnum } from '@fastgpt/core/dataset/constant';
import { getFileIcon, getSpecialFileIcon } from '@fastgpt/common/tools/file';
const FileCard = ({ kbId }: { kbId: string }) => {
......
projects/app/src/pages/kb/detail/components/Import/FileSelect.tsx
......@@ -2,7 +2,8 @@ import MyIcon from '@/components/Icon';
import { useLoading } from '@/web/common/hooks/useLoading';
import { useSelectFile } from '@/web/common/hooks/useSelectFile';
import { useToast } from '@/web/common/hooks/useToast';
import { simpleText, splitText2Chunks } from '@/utils/file';
import { splitText2Chunks } from '@/utils/file';
import { simpleText } from '@fastgpt/common/tools/str';
import {
uploadFiles,
fileDownload,
......
projects/app/src/pages/kb/list/component/EditFolderModal.tsx
......@@ -4,7 +4,7 @@ import MyModal from '@/components/MyModal';
import { useTranslation } from 'react-i18next';
import { useRequest } from '@/web/common/hooks/useRequest';
import { postCreateDataset, putDatasetById } from '@/web/core/api/dataset';
import { FolderAvatarSrc, KbTypeEnum } from '@/constants/dataset';
import { FolderAvatarSrc, DatasetTypeEnum } from '@fastgpt/core/dataset/constant';
const EditFolderModal = ({
onClose,
......@@ -47,7 +47,7 @@ const EditFolderModal = ({
return postCreateDataset({
parentId,
name: val,
type: KbTypeEnum.folder,
type: DatasetTypeEnum.folder,
avatar: FolderAvatarSrc,
tags: []
});
......
projects/app/src/pages/kb/list/component/MoveModal.tsx
......@@ -14,7 +14,7 @@ import Avatar from '@/components/Avatar';
import MyTooltip from '@/components/MyTooltip';
import MyModal from '@/components/MyModal';
import MyIcon from '@/components/Icon';
import { KbTypeEnum } from '@/constants/dataset';
import { DatasetTypeEnum } from '@fastgpt/core/dataset/constant';
import { useTranslation } from 'react-i18next';
import { useQuery } from '@tanstack/react-query';
import { getDatasets, putDatasetById, getDatasetPaths } from '@/web/core/api/dataset';
......@@ -112,7 +112,7 @@ const MoveModal = ({
<MyTooltip
key={item._id}
label={
item.type === KbTypeEnum.dataset
item.type === DatasetTypeEnum.dataset
? t('kb.Select Dataset')
: t('kb.Select Folder')
}
......@@ -142,7 +142,7 @@ const MoveModal = ({
</Box>
</Flex>
<Flex justifyContent={'flex-end'} alignItems={'center'} fontSize={'sm'}>
{item.type === KbTypeEnum.folder ? (
{item.type === DatasetTypeEnum.folder ? (
<Box color={'myGray.500'}>{t('Folder')}</Box>
) : (
<>
......
projects/app/src/pages/kb/list/index.tsx
......@@ -26,7 +26,7 @@ import Avatar from '@/components/Avatar';
import MyIcon from '@/components/Icon';
import { serviceSideProps } from '@/web/common/utils/i18n';
import dynamic from 'next/dynamic';
import { FolderAvatarSrc, KbTypeEnum } from '@/constants/dataset';
import { FolderAvatarSrc, DatasetTypeEnum } from '@fastgpt/core/dataset/constant';
import Tag from '@/components/Tag';
import MyMenu from '@/components/MyMenu';
import { useRequest } from '@/web/common/hooks/useRequest';
......@@ -46,8 +46,8 @@ const Kb = () => {
const { setLoading } = useGlobalStore();
const DeleteTipsMap = useRef({
[KbTypeEnum.folder]: t('kb.deleteFolderTips'),
[KbTypeEnum.dataset]: t('kb.deleteDatasetTips')
[DatasetTypeEnum.folder]: t('kb.deleteFolderTips'),
[DatasetTypeEnum.dataset]: t('kb.deleteDatasetTips')
});
const { openConfirm, ConfirmModal } = useConfirm({
......@@ -224,7 +224,7 @@ const Kb = () => {
border={theme.borders.md}
boxShadow={'none'}
position={'relative'}
data-drag-id={kb.type === KbTypeEnum.folder ? kb._id : undefined}
data-drag-id={kb.type === DatasetTypeEnum.folder ? kb._id : undefined}
borderColor={dragTargetId === kb._id ? 'myBlue.600' : ''}
draggable
onDragStart={(e) => {
......@@ -234,7 +234,7 @@ const Kb = () => {
e.preventDefault();
const targetId = e.currentTarget.getAttribute('data-drag-id');
if (!targetId) return;
KbTypeEnum.folder && setDragTargetId(targetId);
DatasetTypeEnum.folder && setDragTargetId(targetId);
}}
onDragLeave={(e) => {
e.preventDefault();
......@@ -261,14 +261,14 @@ const Kb = () => {
}
}}
onClick={() => {
if (kb.type === KbTypeEnum.folder) {
if (kb.type === DatasetTypeEnum.folder) {
router.push({
pathname: '/kb/list',
query: {
parentId: kb._id
}
});
} else if (kb.type === KbTypeEnum.dataset) {
} else if (kb.type === DatasetTypeEnum.dataset) {
router.push({
pathname: '/kb/detail',
query: {
......@@ -377,7 +377,7 @@ const Kb = () => {
</Flex>
</Box>
<Flex justifyContent={'flex-end'} alignItems={'center'} fontSize={'sm'}>
{kb.type === KbTypeEnum.folder ? (
{kb.type === DatasetTypeEnum.folder ? (
<Box color={'myGray.500'}>{t('Folder')}</Box>
) : (
<>
......
projects/app/src/pages/login/components/RegisterForm.tsx
......@@ -69,7 +69,8 @@ const RegisterForm = ({ setPageType, loginSuccess }: Props) => {
postCreateApp({
avatar: template.avatar,
name: template.name,
modules: template.modules
modules: template.modules,
type: template.type
});
});
}, 100);
......
projects/app/src/service/common/bill/push.ts
import { Bill, User, OutLink } from '@/service/mongo';
import { Bill } from '@/service/mongo';
import { MongoUser } from '@fastgpt/support/user/schema';
import { BillSourceEnum } from '@/constants/user';
import { getModel } from '@/service/utils/data';
import { ChatHistoryItemResType } from '@/types/chat';
......@@ -10,7 +11,7 @@ import { defaultQGModel } from '@/pages/api/system/getInitData';
async function createBill(data: CreateBillType) {
try {
await Promise.all([
User.findByIdAndUpdate(data.userId, {
MongoUser.findByIdAndUpdate(data.userId, {
$inc: { balance: -data.total }
}),
Bill.create(data)
......@@ -50,7 +51,7 @@ async function concatBill({
}
}
),
User.findByIdAndUpdate(userId, {
MongoUser.findByIdAndUpdate(userId, {
$inc: { balance: -total }
})
]);
......@@ -177,12 +178,12 @@ export const pushQuestionGuideBill = ({ tokens, userId }: { tokens: number; user
const total = qgModel.price * tokens;
createBill({
userId,
appName: '问题指引',
appName: '下一步指引',
total,
source: BillSourceEnum.fastgpt,
list: [
{
moduleName: '问题指引',
moduleName: '下一步指引',
amount: total,
model: qgModel.name,
tokenLen: tokens
......
projects/app/src/service/common/bill/schema.ts
import { Schema, model, models, Model } from 'mongoose';
import { connectionMongo, type Model } from '@fastgpt/common/mongo';
const { Schema, model, models } = connectionMongo;
import { BillSchema as BillType } from '@/types/common/bill';
import { BillSourceMap } from '@/constants/user';
......
projects/app/src/service/common/ipLimit/schema.ts deleted 100644 → 0
import { Schema, model, models, Model } from 'mongoose';
import type { IpLimitSchemaType } from '@/types/common/ipLimit';
const IpLimitSchema = new Schema({
eventId: {
type: String,
required: true
},
ip: {
type: String,
required: true
},
account: {
type: Number,
default: 0
},
lastMinute: {
type: Date,
default: () => new Date()
}
});
export const IpLimit: Model<IpLimitSchemaType> =
models['ip_limit'] || model('ip_limit', IpLimitSchema);
projects/app/src/service/dataset/auth.ts
import { isSpecialFileId } from '@fastgpt/core/dataset/utils';
import { GridFSStorage } from '../lib/gridfs';
import { Types } from 'mongoose';
import { Types } from '@fastgpt/common/mongo';
export async function authFileIdValid(fileId?: string) {
if (!fileId) return true;
......
projects/app/src/service/events/generateQA.ts
import { TrainingData } from '@/service/mongo';
import { pushQABill } from '@/service/common/bill/push';
import { TrainingModeEnum } from '@/constants/plugin';
import { ERROR_ENUM } from '../errorCode';
import { ERROR_ENUM } from '@fastgpt/common/constant/errorCode';
import { sendInform } from '@/pages/api/user/inform/send';
import { authBalanceByUid } from '../utils/auth';
import { authBalanceByUid } from '@fastgpt/support/user/auth';
import { getAIApi } from '@fastgpt/core/ai/config';
import type { ChatCompletionRequestMessage } from '@fastgpt/core/ai/type';
import { addLog } from '../utils/tools';
......
projects/app/src/service/events/generateVector.ts
import { insertData2Dataset } from '@/service/pg';
import { getVector } from '@/pages/api/openapi/plugin/vector';
import { TrainingData } from '../models/trainingData';
import { ERROR_ENUM } from '../errorCode';
import { ERROR_ENUM } from '@fastgpt/common/constant/errorCode';
import { TrainingModeEnum } from '@/constants/plugin';
import { sendInform } from '@/pages/api/user/inform/send';
import { addLog } from '../utils/tools';
......
projects/app/src/service/lib/gridfs.ts
import mongoose, { Types } from 'mongoose';
import { Types, connectionMongo } from '@fastgpt/common/mongo';
import fs from 'fs';
import fsp from 'fs/promises';
import { ERROR_ENUM } from '../errorCode';
import { ERROR_ENUM } from '@fastgpt/common/constant/errorCode';
import type { GSFileInfoType } from '@/types/common/file';
enum BucketNameEnum {
......@@ -18,10 +18,10 @@ export class GridFSStorage {
this.uid = String(uid);
}
Collection() {
return mongoose.connection.db.collection(`${this.bucket}.files`);
return connectionMongo.connection.db.collection(`${this.bucket}.files`);
}
GridFSBucket() {
return new mongoose.mongo.GridFSBucket(mongoose.connection.db, {
return new connectionMongo.mongo.GridFSBucket(connectionMongo.connection.db, {
bucketName: this.bucket
});
}
......
projects/app/src/service/models/app.ts
import { Schema, model, models, Model } from 'mongoose';
import { connectionMongo, type Model } from '@fastgpt/common/mongo';
const { Schema, model, models } = connectionMongo;
import { AppSchema as AppType } from '@/types/mongoSchema';
const AppSchema = new Schema({
......
projects/app/src/service/models/chat.ts
import { Schema, model, models, Model } from 'mongoose';
import { connectionMongo, type Model } from '@fastgpt/common/mongo';
const { Schema, model, models } = connectionMongo;
import { ChatSchema as ChatType } from '@/types/mongoSchema';
import { ChatRoleMap, TaskResponseKeyEnum } from '@/constants/chat';
import { ChatSourceMap } from '@/constants/chat';
......
projects/app/src/service/models/chatItem.ts
import { Schema, model, models, Model } from 'mongoose';
import { connectionMongo, type Model } from '@fastgpt/common/mongo';
const { Schema, model, models } = connectionMongo;
import { ChatItemSchema as ChatItemType } from '@/types/mongoSchema';
import { ChatRoleMap, TaskResponseKeyEnum } from '@/constants/chat';
import { customAlphabet } from 'nanoid';
......
projects/app/src/service/models/collection.ts
import { Schema, model, models, Model as MongoModel } from 'mongoose';
import { connectionMongo, type Model } from '@fastgpt/common/mongo';
const { Schema, model, models } = connectionMongo;
import { CollectionSchema as CollectionType } from '@/types/mongoSchema';
const CollectionSchema = new Schema({
......@@ -14,5 +15,5 @@ const CollectionSchema = new Schema({
}
});
export const Collection: MongoModel<CollectionType> =
export const Collection: Model<CollectionType> =
models['collection'] || model('collection', CollectionSchema);
projects/app/src/service/models/image.ts
import { Schema, model, models, Model } from 'mongoose';
import { connectionMongo, type Model } from '@fastgpt/common/mongo';
const { Schema, model, models } = connectionMongo;
const ImageSchema = new Schema({
userId: {
......
projects/app/src/service/models/inform.ts
import { Schema, model, models, Model } from 'mongoose';
import { connectionMongo, type Model } from '@fastgpt/common/mongo';
const { Schema, model, models } = connectionMongo;
import { informSchema } from '@/types/mongoSchema';
import { InformTypeMap } from '@/constants/user';
......
projects/app/src/service/models/pay.ts
import { Schema, model, models, Model } from 'mongoose';
import { connectionMongo, type Model } from '@fastgpt/common/mongo';
const { Schema, model, models } = connectionMongo;
import { PaySchema as PayType } from '@/types/mongoSchema';
const PaySchema = new Schema({
userId: {
......
projects/app/src/service/models/promotionRecord.ts
import { Schema, model, models, Model } from 'mongoose';
import { connectionMongo, type Model } from '@fastgpt/common/mongo';
const { Schema, model, models } = connectionMongo;
import { PromotionRecordSchema as PromotionRecordType } from '@/types/mongoSchema';
const PromotionRecordSchema = new Schema({
......
projects/app/src/service/models/trainingData.ts
/* 模型的知识库 */
import { Schema, model, models, Model as MongoModel } from 'mongoose';
import { connectionMongo, type Model } from '@fastgpt/common/mongo';
const { Schema, model, models } = connectionMongo;
import { TrainingDataSchema as TrainingDateType } from '@/types/mongoSchema';
import { TrainingTypeMap } from '@/constants/plugin';
......@@ -68,5 +69,5 @@ try {
console.log(error);
}
export const TrainingData: MongoModel<TrainingDateType> =
export const TrainingData: Model<TrainingDateType> =
models['trainingData'] || model('trainingData', TrainingDataSchema);
projects/app/src/service/moduleDispatch/agent/classifyQuestion.ts
......@@ -105,7 +105,7 @@ async function functionCall({
required: ['type']
}
};
const ai = getAIApi(user.openaiAccount);
const ai = getAIApi(user.openaiAccount, 48000);
const response = await ai.chat.completions.create({
model: cqModel.model,
......
projects/app/src/service/moduleDispatch/agent/extract.ts
......@@ -126,7 +126,7 @@ async function functionCall({
}
};
const ai = getAIApi(user.openaiAccount);
const ai = getAIApi(user.openaiAccount, 480000);
const response = await ai.chat.completions.create({
model: extractModel.model,
......
projects/app/src/service/moduleDispatch/chat/oneapi.ts
......@@ -10,7 +10,7 @@ import { TaskResponseKeyEnum } from '@/constants/chat';
import { getChatModel } from '@/service/utils/data';
import { countModelPrice } from '@/service/common/bill/push';
import { ChatModelItemType } from '@/types/model';
import { postTextCensor } from '@/service/common/api/plugins';
import { postTextCensor } from '@fastgpt/common/plusApi/censor';
import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/core/ai/constant';
import { AppModuleItemType } from '@/types/app';
import { countMessagesTokens, sliceMessagesTB } from '@/utils/common/tiktoken';
......@@ -151,7 +151,7 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise<ChatResp
};
} else {
const unStreamResponse = response as ChatCompletion;
const answer = unStreamResponse.choices?.[0].message?.content || '';
const answer = unStreamResponse.choices?.[0]?.message?.content || '';
const totalTokens = unStreamResponse.usage?.total_tokens || 0;
const completeMessages = filterMessages.concat({
......
projects/app/src/service/moduleDispatch/kb/search.ts
......@@ -24,6 +24,7 @@ export type KBSearchResponse = {
export async function dispatchKBSearch(props: Record<string, any>): Promise<KBSearchResponse> {
const {
moduleName,
user,
inputs: { kbList = [], similarity = 0.4, limit = 5, userChatInput }
} = props as KBSearchProps;
......@@ -45,10 +46,10 @@ export async function dispatchKBSearch(props: Record<string, any>): Promise<KBSe
// search kb
const res: any = await PgClient.query(
`BEGIN;
SET LOCAL ivfflat.probes = ${global.systemEnv.pgIvfflatProbe || 10};
SET LOCAL hnsw.ef_search = ${global.systemEnv.pgHNSWEfSearch || 40};
select id, kb_id, q, a, source, file_id, (vector <#> '[${
vectors[0]
}]') * -1 AS score from ${PgDatasetTableName} where kb_id IN (${kbList
}]') * -1 AS score from ${PgDatasetTableName} where user_id='${user._id}' AND kb_id IN (${kbList
.map((item) => `'${item.kbId}'`)
.join(',')}) AND vector <#> '[${vectors[0]}]' < -${similarity} order by vector <#> '[${
vectors[0]
......
projects/app/src/service/mongo.ts
import mongoose from 'mongoose';
import { startQueue } from './utils/tools';
import { getInitConfig } from '@/pages/api/system/getInitData';
import { User } from './models/user';
import { PRICE_SCALE } from '@fastgpt/common/bill/constants';
import { initPg } from './pg';
import { createHashPassword } from '@/utils/tools';
import { createLogger, format, transports } from 'winston';
import 'winston-mongodb';
import { getTikTokenEnc } from '@/utils/common/tiktoken';
import { initHttpAgent } from '@fastgpt/core/init';
import { MongoUser } from '@fastgpt/support/user/schema';
import { connectMongo } from '@fastgpt/common/mongo/init';
import { hashStr } from '@fastgpt/common/tools/str';
import { getInitConfig, initGlobal } from '@/pages/api/system/getInitData';
/**
* connect MongoDB and init data
*/
export async function connectToDatabase(): Promise<void> {
if (global.mongodb) {
return;
}
global.mongodb = 'connecting';
// init global data
global.qaQueueLen = 0;
global.vectorQueueLen = 0;
global.sendInformQueue = [];
global.sendInformQueueLen = 0;
// logger
initLogger();
// init function
getInitConfig();
// init tikToken
getTikTokenEnc();
initHttpAgent();
try {
mongoose.set('strictQuery', true);
global.mongodb = await mongoose.connect(process.env.MONGODB_URI as string, {
bufferCommands: true,
maxConnecting: Number(process.env.DB_MAX_LINK || 5),
maxPoolSize: Number(process.env.DB_MAX_LINK || 5),
minPoolSize: 2
});
await initRootUser();
initPg();
console.log('mongo connected');
} catch (error) {
console.log('error->', 'mongo connect error');
global.mongodb = null;
}
// init function
startQueue();
}
function initLogger() {
global.logger = createLogger({
transports: [
new transports.MongoDB({
db: process.env.MONGODB_URI as string,
collection: 'server_logs',
options: {
useUnifiedTopology: true
},
cappedSize: 500000000,
tryReconnect: true,
metaKey: 'meta',
format: format.combine(format.timestamp(), format.json())
}),
new transports.Console({
format: format.combine(
format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
format.printf((info) => {
if (info.level === 'error') {
console.log(info.meta);
return `[${info.level.toLocaleUpperCase()}]: ${[info.timestamp]}: ${info.message}`;
}
return `[${info.level.toLocaleUpperCase()}]: ${[info.timestamp]}: ${info.message}${
info.meta ? `: ${JSON.stringify(info.meta)}` : ''
}`;
})
)
})
]
await connectMongo({
beforeHook: () => {
initGlobal();
getInitConfig();
},
afterHook: async () => {
await initRootUser();
initPg();
// start queue
startQueue();
}
});
}
async function initRootUser() {
try {
const rootUser = await User.findOne({
const rootUser = await MongoUser.findOne({
username: 'root'
});
const psw = process.env.DEFAULT_ROOT_PSW || '123456';
if (rootUser) {
await User.findOneAndUpdate(
await MongoUser.findOneAndUpdate(
{ username: 'root' },
{
password: createHashPassword(psw),
password: hashStr(psw),
balance: 999999 * PRICE_SCALE
}
);
} else {
await User.create({
await MongoUser.create({
username: 'root',
password: createHashPassword(psw),
password: hashStr(psw),
balance: 999999 * PRICE_SCALE
});
}
......@@ -121,15 +59,10 @@ async function initRootUser() {
export * from './models/chat';
export * from './models/chatItem';
export * from './models/app';
export * from './models/user';
export * from './common/bill/schema';
export * from './models/pay';
export * from './models/trainingData';
export * from './models/promotionRecord';
export * from './models/collection';
export * from './models/kb';
export * from './models/inform';
export * from './models/image';
export * from './support/outLink/schema';
export * from './support/openapi/schema';
projects/app/src/service/pg.ts
......@@ -228,6 +228,7 @@ export async function initPg() {
q TEXT NOT NULL,
a TEXT
);
CREATE INDEX IF NOT EXISTS vector_index ON ${PgDatasetTableName} USING hnsw (vector vector_ip_ops) WITH (m = 16, ef_construction = 64);
`);
console.log('init pg successful');
} catch (error) {
......
projects/app/src/service/response.ts
import { sseResponseEventEnum } from '@/constants/chat';
import { NextApiResponse } from 'next';
import { proxyError, ERROR_RESPONSE, ERROR_ENUM } from './errorCode';
import { proxyError, ERROR_RESPONSE, ERROR_ENUM } from '@fastgpt/common/constant/errorCode';
import { clearCookie, sseResponse, addLog } from './utils/tools';
export interface ResponseType<T = any> {
......
projects/app/src/service/support/outLink/auth.ts deleted 100644 → 0
import { PRICE_SCALE } from '@fastgpt/common/bill/constants';
import { IpLimit } from '@/service/common/ipLimit/schema';
import { authBalanceByUid, AuthUserTypeEnum } from '@/service/utils/auth';
import { OutLinkSchema } from '@/types/support/outLink';
import { OutLink } from './schema';
import axios from 'axios';
type AuthLinkProps = { ip?: string | null; authToken?: string; question: string };
export async function authOutLinkChat({
shareId,
ip,
authToken,
question
}: AuthLinkProps & {
shareId: string;
}) {
// get outLink
const outLink = await OutLink.findOne({
shareId
});
if (!outLink) {
return Promise.reject('分享链接无效');
}
const uid = String(outLink.userId);
const [user] = await Promise.all([
authBalanceByUid(uid), // authBalance
...(global.feConfigs?.isPlus ? [authOutLinkLimit({ outLink, ip, authToken, question })] : []) // limit auth
]);
return {
user,
userId: String(outLink.userId),
appId: String(outLink.appId),
authType: AuthUserTypeEnum.token,
responseDetail: outLink.responseDetail
};
}
export async function authOutLinkLimit({
outLink,
ip,
authToken,
question
}: AuthLinkProps & {
outLink: OutLinkSchema;
}) {
if (!ip || !outLink.limit) {
return;
}
if (outLink.limit.expiredTime && outLink.limit.expiredTime.getTime() < Date.now()) {
return Promise.reject('分享链接已过期');
}
if (outLink.limit.credit > -1 && outLink.total > outLink.limit.credit * PRICE_SCALE) {
return Promise.reject('链接超出使用限制');
}
// ip limit
await (async () => {
if (!outLink.limit) {
return;
}
try {
const ipLimit = await IpLimit.findOne({ ip, eventId: outLink._id });
// first request
if (!ipLimit) {
return await IpLimit.create({
eventId: outLink._id,
ip,
account: outLink.limit.QPM - 1
});
}
// over one minute
const diffTime = Date.now() - ipLimit.lastMinute.getTime();
if (diffTime >= 60 * 1000) {
ipLimit.account = outLink.limit.QPM - 1;
ipLimit.lastMinute = new Date();
return await ipLimit.save();
}
// over limit
if (ipLimit.account <= 0) {
return Promise.reject(
`每分钟仅能请求 ${outLink.limit.QPM} 次, ${60 - Math.round(diffTime / 1000)}s 后重试~`
);
}
// update limit
ipLimit.account = ipLimit.account - 1;
await ipLimit.save();
} catch (error) {}
})();
// url auth. send request
await authShareStart({ authToken, tokenUrl: outLink.limit.hookUrl, question });
}
export async function authOutLinkId({ id }: { id: string }) {
const outLink = await OutLink.findOne({
shareId: id
});
if (!outLink) {
return Promise.reject('分享链接无效');
}
return {
userId: String(outLink.userId)
};
}
type TokenAuthResponseType = {
success: boolean;
msg?: string;
message?: string;
};
export const authShareChatInit = async (authToken?: string, tokenUrl?: string) => {
if (!tokenUrl || !global.feConfigs?.isPlus) return;
try {
const { data } = await axios<TokenAuthResponseType>({
baseURL: tokenUrl,
url: '/shareAuth/init',
method: 'POST',
data: {
token: authToken
}
});
if (data?.success !== true) {
return Promise.reject(data?.message || data?.msg || '身份校验失败');
}
} catch (error) {
return Promise.reject('身份校验失败');
}
};
export const authShareStart = async ({
tokenUrl,
authToken,
question
}: {
authToken?: string;
question: string;
tokenUrl?: string;
}) => {
if (!tokenUrl || !global.feConfigs?.isPlus) return;
try {
const { data } = await axios<TokenAuthResponseType>({
baseURL: tokenUrl,
url: '/shareAuth/start',
method: 'POST',
data: {
token: authToken,
question
}
});
if (data?.success !== true) {
return Promise.reject(data?.message || data?.msg || '身份校验失败');
}
} catch (error) {
return Promise.reject('身份校验失败');
}
};
projects/app/src/service/utils/auth.ts
import type { NextApiRequest } from 'next';
import Cookie from 'cookie';
import { App, User, KB } from '../mongo';
import type { AppSchema, UserModelSchema } from '@/types/mongoSchema';
import { ERROR_ENUM } from '../errorCode';
import { authJWT } from './tools';
import { authOpenApiKey } from '../support/openapi/auth';
import { authOutLinkId } from '../support/outLink/auth';
export enum AuthUserTypeEnum {
token = 'token',
root = 'root',
apikey = 'apikey',
outLink = 'outLink'
}
/* auth balance */
export const authBalanceByUid = async (uid: string) => {
const user = await User.findById<UserModelSchema>(
uid,
'_id username balance openaiAccount timezone'
);
if (!user) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
if (user.balance <= 0) {
return Promise.reject(ERROR_ENUM.insufficientQuota);
}
return user;
};
/* uniform auth user */
export const authUser = async ({
req,
authToken = false,
authRoot = false,
authApiKey = false,
authBalance = false,
authOutLink
}: {
req: NextApiRequest;
authToken?: boolean;
authRoot?: boolean;
authApiKey?: boolean;
authBalance?: boolean;
authOutLink?: boolean;
}) => {
const authCookieToken = async (cookie?: string, token?: string): Promise<string> => {
// 获取 cookie
const cookies = Cookie.parse(cookie || '');
const cookieToken = cookies.token || token;
if (!cookieToken) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
return await authJWT(cookieToken);
};
// from authorization get apikey
const parseAuthorization = async (authorization?: string) => {
if (!authorization) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
// Bearer fastgpt-xxxx-appId
const auth = authorization.split(' ')[1];
if (!auth) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
const { apikey, appId: authorizationAppid = '' } = await (async () => {
const arr = auth.split('-');
// abandon
if (arr.length === 3) {
return {
apikey: `${arr[0]}-${arr[1]}`,
appId: arr[2]
};
}
if (arr.length === 2) {
return {
apikey: auth
};
}
return Promise.reject(ERROR_ENUM.unAuthorization);
})();
// auth apikey
const { userId, appId: apiKeyAppId = '' } = await authOpenApiKey({ apikey });
return {
uid: userId,
apikey,
appId: apiKeyAppId || authorizationAppid
};
};
// root user
const parseRootKey = async (rootKey?: string, userId = '') => {
if (!rootKey || !process.env.ROOT_KEY || rootKey !== process.env.ROOT_KEY) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
return userId;
};
const { cookie, token, apikey, rootkey, userid, authorization } = (req.headers || {}) as {
cookie?: string;
token?: string;
apikey?: string;
rootkey?: string; // abandon
userid?: string;
authorization?: string;
};
const { shareId } = (req?.body || {}) as { shareId?: string };
let uid = '';
let appId = '';
let openApiKey = apikey;
let authType: `${AuthUserTypeEnum}` = AuthUserTypeEnum.token;
if (authOutLink && shareId) {
const res = await authOutLinkId({ id: shareId });
uid = res.userId;
authType = AuthUserTypeEnum.outLink;
} else if (authToken && (cookie || token)) {
// user token(from fastgpt web)
uid = await authCookieToken(cookie, token);
authType = AuthUserTypeEnum.token;
} else if (authRoot && rootkey) {
// root user
uid = await parseRootKey(rootkey, userid);
authType = AuthUserTypeEnum.root;
} else if (authApiKey && apikey) {
// apikey
const parseResult = await authOpenApiKey({ apikey });
uid = parseResult.userId;
authType = AuthUserTypeEnum.apikey;
openApiKey = parseResult.apikey;
} else if (authApiKey && authorization) {
// apikey from authorization
const authResponse = await parseAuthorization(authorization);
uid = authResponse.uid;
appId = authResponse.appId;
openApiKey = authResponse.apikey;
authType = AuthUserTypeEnum.apikey;
}
// not rootUser and no uid, reject request
if (!rootkey && !uid) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
// balance check
const user = await (() => {
if (authBalance) {
return authBalanceByUid(uid);
}
})();
return {
userId: String(uid),
appId,
authType,
user,
apikey: openApiKey
};
};
import { App } from '../mongo';
import { MongoDataset } from '@fastgpt/core/dataset/schema';
import type { AppSchema } from '@/types/mongoSchema';
import { ERROR_ENUM } from '@fastgpt/common/constant/errorCode';
// 模型使用权校验
export const authApp = async ({
......@@ -200,8 +37,8 @@ export const authApp = async ({
};
// 知识库操作权限
export const authKb = async ({ kbId, userId }: { kbId: string; userId: string }) => {
const kb = await KB.findOne({
export const authDataset = async ({ kbId, userId }: { kbId: string; userId: string }) => {
const kb = await MongoDataset.findOne({
_id: kbId,
userId
});
......
projects/app/src/service/utils/tools.ts
import type { NextApiResponse, NextApiHandler, NextApiRequest } from 'next';
import NextCors from 'nextjs-cors';
import crypto from 'crypto';
import jwt from 'jsonwebtoken';
import { generateQA } from '../events/generateQA';
import { generateVector } from '../events/generateVector';
import { ERROR_ENUM } from '../errorCode';
/* 密码加密 */
export const hashPassword = (psw: string) => {
return crypto.createHash('sha256').update(psw).digest('hex');
};
/* 生成 token */
export const generateToken = (userId: string) => {
const key = process.env.TOKEN_KEY as string;
const token = jwt.sign(
{
userId,
exp: Math.floor(Date.now() / 1000) + 60 * 60 * 24 * 7
},
key
);
return token;
};
// auth token
export const authJWT = (token: string) =>
new Promise<string>((resolve, reject) => {
const key = process.env.TOKEN_KEY as string;
jwt.verify(token, key, function (err, decoded: any) {
if (err || !decoded?.userId) {
reject(ERROR_ENUM.unAuthorization);
return;
}
resolve(decoded.userId);
});
});
/* set cookie */
export const setCookie = (res: NextApiResponse, token: string) => {
res.setHeader(
......@@ -67,6 +34,7 @@ export function withNextCors(handler: NextApiHandler): NextApiHandler {
/* start task */
export const startQueue = () => {
if (!global.systemEnv) return;
for (let i = 0; i < global.systemEnv.qaMaxProcess; i++) {
generateQA();
}
......
projects/app/src/types/app.d.ts
......@@ -11,7 +11,7 @@ import type {
FlowOutputItemType,
FlowOutputTargetItemType
} from '@/types/core/app/flow';
import type { AppSchema, ChatSchema, kbSchema } from './mongoSchema';
import type { AppSchema, ChatSchema } from './mongoSchema';
import { ChatModelType } from '@/constants/model';
import { FlowValueTypeEnum } from '@/constants/flow';
......
projects/app/src/types/common/ipLimit.d.ts deleted 100644 → 0
export type IpLimitSchemaType = {
_id: string;
eventId: string;
ip: string;
account: number;
lastMinute: Date;
};
projects/app/src/types/core/chat/type.d.ts
import type { ChatCompletionRequestMessage } from '@fastgpt/core/ai/type';
import type { NextApiResponse } from 'next';
import { RunningModuleItemType } from '@/types/app';
import { UserModelSchema } from '@/types/mongoSchema';
import type { UserModelSchema } from '@fastgpt/support/user/type.d';
export type MessageItemType = ChatCompletionRequestMessage & { dataId?: string; content: string };
......
projects/app/src/types/core/dataset/file.d.ts
import { FileStatusEnum } from '@/constants/dataset';
import { FileStatusEnum } from '@fastgpt/core/dataset/constant';
export type DatasetFileItemType = {
id: string;
......
projects/app/src/types/core/dataset/index.d.ts
import { FileStatusEnum } from '@/constants/dataset';
import { PgDataItemType } from './data';
import { VectorModelItemType } from '../../model';
import type { kbSchema } from '../../mongoSchema';
import type { DatasetSchemaType } from '@fastgpt/core/dataset/type.d';
export type DatasetsItemType = Omit<kbSchema, 'vectorModel'> & {
export type DatasetsItemType = Omit<DatasetSchemaType, 'vectorModel'> & {
vectorModel: VectorModelItemType;
};
......
projects/app/src/types/index.d.ts
import type { Mongoose } from 'mongoose';
import type { Pool } from 'pg';
import type { Tiktoken } from 'js-tiktoken';
import type { Logger } from 'winston';
import {
ChatModelItemType,
FunctionModelItemType,
......@@ -19,59 +17,31 @@ export type PagingData<T> = {
export type RequestPaging = { pageNum: number; pageSize: number; [key]: any };
export type FeConfigsType = {
show_emptyChat?: boolean;
show_register?: boolean;
show_appStore?: boolean;
show_contact?: boolean;
show_git?: boolean;
show_doc?: boolean;
show_pay?: boolean;
show_openai_account?: boolean;
show_promotion?: boolean;
hide_app_flow?: boolean;
openAPIUrl?: string;
systemTitle?: string;
authorText?: string;
googleClientVerKey?: string;
isPlus?: boolean;
oauth?: {
github?: string;
google?: string;
};
limit?: {
exportLimitMinutes?: number;
};
scripts?: { [key: string]: string }[];
};
export type SystemEnvType = {
pluginBaseUrl?: string;
openapiPrefix?: string;
vectorMaxProcess: number;
qaMaxProcess: number;
pgIvfflatProbe: number;
pgHNSWEfSearch: number;
};
declare global {
var mongodb: Mongoose | string | null;
var pgClient: Pool | null;
var qaQueueLen: number;
var vectorQueueLen: number;
var TikToken: Tiktoken;
var logger: Logger;
var sendInformQueue: (() => Promise<void>)[];
var sendInformQueueLen: number;
var feConfigs: FeConfigsType;
var systemEnv: SystemEnvType;
var vectorModels: VectorModelItemType[];
var chatModels: ChatModelItemType[];
var qaModel: QAModelItemType;
var extractModel: FunctionModelItemType;
var cqModel: FunctionModelItemType;
var qgModel: FunctionModelItemType;
var vectorModels: VectorModelItemType[];
var priceMd: string;
var systemVersion: string;
interface Window {
......
projects/app/src/types/mongoSchema.d.ts
......@@ -6,27 +6,6 @@ import { TrainingModeEnum } from '@/constants/plugin';
import type { AppModuleItemType } from './app';
import { ChatSourceEnum } from '@/constants/chat';
import { AppTypeEnum } from '@/constants/app';
import { KbTypeEnum } from '@/constants/dataset';
export interface UserModelSchema {
_id: string;
username: string;
password: string;
avatar: string;
balance: number;
promotionRate: number;
inviterId?: string;
openaiKey: string;
createTime: number;
timezone: string;
openaiAccount?: {
key: string;
baseUrl: string;
};
limit: {
exportKbTime?: Date;
};
}
export interface AuthCodeSchema {
_id: string;
......@@ -121,18 +100,6 @@ export interface PromotionRecordSchema {
amount: number;
}
export type kbSchema = {
_id: string;
userId: string;
parentId: string;
updateTime: Date;
avatar: string;
name: string;
vectorModel: string;
tags: string[];
type: `${KbTypeEnum}`;
};
export interface informSchema {
_id: string;
userId: string;
......
projects/app/src/types/user.d.ts
import { BillSourceEnum } from '@/constants/user';
import type { UserModelSchema } from './mongoSchema';
import type { UserModelSchema } from '@fastgpt/support/user/type.d';
import type { BillSchema } from '@/types/common/bill';
export interface UserType {
......
projects/app/src/utils/file.ts
......@@ -7,25 +7,63 @@ import { countPromptTokens } from './common/tiktoken';
* overlapLen - The size of the before and after Text
* maxLen > overlapLen
*/
export const splitText2Chunks = ({ text, maxLen }: { text: string; maxLen: number }) => {
const overlapLen = Math.floor(maxLen * 0.25); // Overlap length
export const splitText2Chunks = ({ text = '', maxLen }: { text: string; maxLen: number }) => {
const overlapLen = Math.floor(maxLen * 0.2); // Overlap length
const tempMarker = 'SPLIT_HERE_SPLIT_HERE';
try {
const tempMarker = 'SPLIT_HERE';
text = text.replace(/\n{3,}/g, '\n');
text = text.replace(/\s/g, ' ');
text = text.replace('\n\n', '');
const stepReg: Record<number, RegExp> = {
0: /(\n)/g,
1: /([。]|\.\s)/g,
2: /([！？]|!\s|\?\s)/g,
3: /([；]|;\s)/g,
4: /([，]|,\s)/g
};
const splitTextRecursively = ({ text = '', step }: { text: string; step: number }) => {
if (text.length <= maxLen) {
return [text];
}
const reg = stepReg[step];
if (!reg) {
// use slice-maxLen to split text
const chunks: string[] = [];
let chunk = '';
for (let i = 0; i < text.length; i += maxLen - overlapLen) {
chunk = text.slice(i, i + maxLen);
chunks.push(chunk);
}
return chunks;
}
// split text by delimiters
const splitTexts = text
.replace(/([。！？；]|\.\s|!\s|\?\s|;\s|\n)/g, `$1${tempMarker}`)
.split(tempMarker)
.replace(reg, `$1${tempMarker}`)
.split(`${tempMarker}`)
.filter((part) => part);
const chunks: string[] = [];
let chunks: string[] = [];
let preChunk = '';
let chunk = '';
for (let i = 0; i < splitTexts.length; i++) {
const text = splitTexts[i];
let text = splitTexts[i];
// chunk over size
if (text.length > maxLen) {
const innerChunks = splitTextRecursively({ text, step: step + 1 });
if (innerChunks.length === 0) continue;
// If the last chunk is too small, it is merged into the next chunk
if (innerChunks[innerChunks.length - 1].length <= maxLen * 0.5) {
text = innerChunks.pop() || '';
chunks = chunks.concat(innerChunks);
} else {
chunks = chunks.concat(innerChunks);
continue;
}
}
chunk += text;
// size over lapLen, push it to next chunk
if (chunk.length > maxLen - overlapLen) {
preChunk += text;
}
......@@ -36,9 +74,14 @@ export const splitText2Chunks = ({ text, maxLen }: { text: string; maxLen: numbe
}
}
if (chunk) {
if (chunk && !chunks[chunks.length - 1].endsWith(chunk)) {
chunks.push(chunk);
}
return chunks;
};
try {
const chunks = splitTextRecursively({ text, step: 0 });
const tokens = chunks.reduce((sum, chunk) => sum + countPromptTokens(chunk, 'system'), 0);
......@@ -50,14 +93,3 @@ export const splitText2Chunks = ({ text, maxLen }: { text: string; maxLen: numbe
throw new Error(getErrText(err));
}
};
/* simple text, remove chinese space and extra \n */
export const simpleText = (text: string) => {
text = text.replace(/([\u4e00-\u9fa5])\s+([\u4e00-\u9fa5])/g, '$1$2');
text = text.replace(/\n{2,}/g, '\n');
text = text.replace(/\s{2,}/g, ' ');
text = text.replace(/[\x00-\x08]/g, ' ');
return text;
};
projects/app/src/utils/tools.ts
import crypto from 'crypto';
import dayjs from 'dayjs';
/**
* 密码加密
*/
export const createHashPassword = (text: string) => {
const hash = crypto.createHash('sha256').update(text).digest('hex');
return hash;
};
/**
* 对象转成 query 字符串
*/
export const Obj2Query = (obj: Record<string, string | number>) => {
......
projects/app/src/web/common/api/bill.ts
......@@ -16,10 +16,10 @@ export const getPayCode = (amount: number) =>
GET<{
codeUrl: string;
payId: string;
}>(`/plusApi/user/pay/getPayCode`, { amount });
}>(`/plusApi/support/user/pay/getPayCode`, { amount });
export const checkPayResult = (payId: string) =>
GET<number>(`/plusApi/user/pay/checkPayResult`, { payId }).then(() => {
GET<number>(`/plusApi/support/user/pay/checkPayResult`, { payId }).then(() => {
try {
GET('/user/account/paySuccess');
} catch (error) {}
......
projects/app/src/web/common/api/request.ts
......@@ -5,7 +5,7 @@ import axios, {
AxiosProgressEvent
} from 'axios';
import { clearToken, getToken } from '@/utils/user';
import { TOKEN_ERROR_CODE } from '@/service/errorCode';
import { TOKEN_ERROR_CODE } from '@fastgpt/common/constant/errorCode';
interface ConfigType {
headers?: { [key: string]: string };
......
projects/app/src/web/common/api/system.ts
......@@ -12,6 +12,7 @@ export const postUploadFiles = (
onUploadProgress: (progressEvent: AxiosProgressEvent) => void
) =>
POST<string[]>('/system/file/upload', data, {
timeout: 60000,
onUploadProgress,
headers: {
'Content-Type': 'multipart/form-data; charset=utf-8'
......
projects/app/src/web/common/store/static.ts
......@@ -6,9 +6,8 @@ import {
import type { InitDateResponse } from '@/global/common/api/systemRes';
import { getSystemInitData } from '@/web/common/api/system';
import { delay } from '@/utils/tools';
import { FeConfigsType } from '@/types';
import type { FeConfigsType } from '@fastgpt/common/type/index.d';
export let systemVersion = '0.0.0';
export let chatModelList: ChatModelItemType[] = [];
export let qaModel: QAModelItemType = {
model: 'gpt-3.5-turbo-16k',
......@@ -18,6 +17,8 @@ export let qaModel: QAModelItemType = {
};
export let vectorModelList: VectorModelItemType[] = [];
export let feConfigs: FeConfigsType = {};
export let priceMd = '';
export let systemVersion = '0.0.0';
let retryTimes = 3;
......@@ -29,6 +30,7 @@ export const clientInitData = async (): Promise<InitDateResponse> => {
qaModel = res.qaModel;
vectorModelList = res.vectorModels;
feConfigs = res.feConfigs;
priceMd = res.priceMd;
systemVersion = res.systemVersion;
return res;
......
projects/app/src/web/core/api/dataset.ts
......@@ -12,7 +12,7 @@ import type {
GetDatasetDataListProps
} from '@/global/core/api/datasetReq.d';
import type { SearchTestResponseType, PushDataResponse } from '@/global/core/api/datasetRes.d';
import { KbTypeEnum } from '@/constants/dataset';
import { DatasetTypeEnum } from '@fastgpt/core/dataset/constant';
import type { DatasetFileItemType } from '@/types/core/dataset/file';
import type { GSFileInfoType } from '@/types/common/file';
import type { QuoteItemType } from '@/types/chat';
......@@ -21,7 +21,7 @@ import download from 'downloadjs';
import type { DatasetDataItemType } from '@/types/core/dataset/data';
/* ======================== dataset ======================= */
export const getDatasets = (data: { parentId?: string; type?: `${KbTypeEnum}` }) =>
export const getDatasets = (data: { parentId?: string; type?: `${DatasetTypeEnum}` }) =>
GET<DatasetsItemType[]>(`/core/dataset/list`, data);
/**
......
projects/app/src/web/support/api/openapi.ts
import { GET, POST, DELETE } from '@/web/common/api/request';
import type { EditApiKeyProps, GetApiKeyProps } from '@/global/support/api/openapiReq.d';
import type { OpenApiSchema } from '@/types/support/openapi';
import type { OpenApiSchema } from '@fastgpt/support/openapi/type.d';
/**
* crete a api key
......
projects/app/src/web/support/api/outLink.ts
import { GET, POST, DELETE } from '@/web/common/api/request';
import type { InitShareChatResponse } from '@/global/support/api/outLinkRes.d';
import type { OutLinkEditType } from '@/types/support/outLink';
import type { OutLinkSchema } from '@/types/support/outLink';
import type { OutLinkEditType, OutLinkSchema } from '@fastgpt/support/outLink/type.d';
/**
* 初始化分享聊天
......
projects/app/src/web/support/api/user.ts
import { GET, POST, PUT } from '@/web/common/api/request';
import { createHashPassword } from '@/utils/tools';
import { hashStr } from '@fastgpt/common/tools/str';
import type { ResLogin, PromotionRecordType } from '@/global/support/api/userRes.d';
import { UserAuthTypeEnum } from '@/constants/common';
import { UserType, UserUpdateParams } from '@/types/user';
......@@ -11,7 +11,7 @@ export const sendAuthCode = (data: {
username: string;
type: `${UserAuthTypeEnum}`;
googleToken: string;
}) => POST(`/plusApi/user/inform/sendAuthCode`, data);
}) => POST(`/plusApi/support/user/inform/sendAuthCode`, data);
export const getTokenLogin = () => GET<UserType>('/user/account/tokenLogin');
export const oauthLogin = (params: {
......@@ -19,7 +19,7 @@ export const oauthLogin = (params: {
code: string;
callbackUrl: string;
inviterId?: string;
}) => POST<ResLogin>('/plusApi/user/account/login/oauth', params);
}) => POST<ResLogin>('/plusApi/support/user/account/login/oauth', params);
export const postRegister = ({
username,
......@@ -32,11 +32,11 @@ export const postRegister = ({
password: string;
inviterId?: string;
}) =>
POST<ResLogin>(`/plusApi/user/account/register/emailAndPhone`, {
POST<ResLogin>(`/plusApi/support/user/account/register/emailAndPhone`, {
username,
code,
inviterId,
password: createHashPassword(password)
password: hashStr(password)
});
export const postFindPassword = ({
......@@ -48,22 +48,22 @@ export const postFindPassword = ({
code: string;
password: string;
}) =>
POST<ResLogin>(`/plusApi/user/account/password/updateByCode`, {
POST<ResLogin>(`/plusApi/support/user/account/password/updateByCode`, {
username,
code,
password: createHashPassword(password)
password: hashStr(password)
});
export const updatePasswordByOld = ({ oldPsw, newPsw }: { oldPsw: string; newPsw: string }) =>
POST('/user/account/updatePasswordByOld', {
oldPsw: createHashPassword(oldPsw),
newPsw: createHashPassword(newPsw)
oldPsw: hashStr(oldPsw),
newPsw: hashStr(newPsw)
});
export const postLogin = ({ username, password }: { username: string; password: string }) =>
POST<ResLogin>('/user/account/loginByPassword', {
username,
password: createHashPassword(password)
password: hashStr(password)
});
export const loginOut = () => GET('/user/account/loginout');
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment