Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit dd8f2744 by Archer Committed by GitHub
Options

Extraction schema (#398)

parent 7db8d3ea
Showing with 1139 additions and 1091 deletions
Dockerfile
...@@ -9,8 +9,7 @@ ARG name ...@@ -9,8 +9,7 @@ ARG name
# copy packages and one project # copy packages and one project
COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./ COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./
COPY ./packages ./packages COPY ./packages ./packages
COPY ./projects/$name/package.json ./projects/$name/package.json COPY ./projects/$name ./projects/$name
COPY ./projects/$name/pnpm-lock.yaml ./projects/$name/pnpm-lock.yaml
RUN \ RUN \
[ -f pnpm-lock.yaml ] && pnpm install || \ [ -f pnpm-lock.yaml ] && pnpm install || \
...@@ -27,10 +26,7 @@ ARG name ...@@ -27,10 +26,7 @@ ARG name
# copy common node_modules and one project node_modules # copy common node_modules and one project node_modules
COPY --from=deps /app/node_modules ./node_modules COPY --from=deps /app/node_modules ./node_modules
COPY --from=deps /app/packages ./packages COPY --from=deps /app/packages ./packages
COPY ./projects/$name ./projects/$name COPY --from=deps /app/projects/$name ./projects/$name
COPY --from=deps /app/projects/$name/node_modules ./projects/$name/node_modules
COPY pnpm-lock.yaml pnpm-workspace.yaml ./
COPY ./packages ./packages
# Uncomment the following line in case you want to disable telemetry during the build. # Uncomment the following line in case you want to disable telemetry during the build.
ENV NEXT_TELEMETRY_DISABLED 1 ENV NEXT_TELEMETRY_DISABLED 1
......
docSite/content/docs/development/configuration.md
...@@ -23,7 +23,7 @@ weight: 520 ...@@ -23,7 +23,7 @@ weight: 520
"SystemParams": { "SystemParams": {
"vectorMaxProcess": 15, // 向量生成最大进程，结合数据库性能和 key 来设置 "vectorMaxProcess": 15, // 向量生成最大进程，结合数据库性能和 key 来设置
"qaMaxProcess": 15, // QA 生成最大进程，结合数据库性能和 key 来设置 "qaMaxProcess": 15, // QA 生成最大进程，结合数据库性能和 key 来设置
"pgIvfflatProbe": 20 // pg vector 搜索探针。没有设置索引前可忽略，通常 50w 组以上才需要设置。 "pgHNSWEfSearch": 40 // pg vector 索引参数，越大精度高但速度慢
}, },
"ChatModels": [ "ChatModels": [
{ {
......
docSite/content/docs/development/intro.md
...@@ -66,7 +66,7 @@ git clone git@github.com:<github_username>/FastGPT.git ...@@ -66,7 +66,7 @@ git clone git@github.com:<github_username>/FastGPT.git
- `vectorMaxProcess`: 向量生成最大进程，根据数据库和 key 的并发数来决定，通常单个 120 号，2c4g 服务器设置 10~15。 - `vectorMaxProcess`: 向量生成最大进程，根据数据库和 key 的并发数来决定，通常单个 120 号，2c4g 服务器设置 10~15。
- `qaMaxProcess`: QA 生成最大进程 - `qaMaxProcess`: QA 生成最大进程
- `pgIvfflatProbe`: PostgreSQL vector 搜索探针，没有添加 vector 索引时可忽略。 - `pgHNSWEfSearch`: PostgreSQL vector 索引参数，越大搜索精度越高但是速度越慢，具体可看 pgvector 官方说明。
### 5. 运行 ### 5. 运行
......
docSite/content/docs/installation/docker.md
...@@ -72,7 +72,7 @@ brew install orbstack ...@@ -72,7 +72,7 @@ brew install orbstack
依次执行下面命令，创建 FastGPT 文件并拉取`docker-compose.yml`和`config.json`，执行完后目录下会有 2 个文件。 依次执行下面命令，创建 FastGPT 文件并拉取`docker-compose.yml`和`config.json`，执行完后目录下会有 2 个文件。
非 Linux 环境或无法访问外网环境，可手动创建一个目录，并下载下面2个链接的文件。 非 Linux 环境或无法访问外网环境，可手动创建一个目录，并下载下面2个链接的文件: [docker-compose.yml](https://github.com/labring/FastGPT/blob/main/files/deploy/fastgpt/docker-compose.yml),[config.json](https://github.com/labring/FastGPT/blob/main/projects/app/data/config.json)
**注意: `docker-compose.yml` 配置文件中 Mongo 为 5.x，部分服务器不支持，需手动更改其镜像版本为 4.4.24** **注意: `docker-compose.yml` 配置文件中 Mongo 为 5.x，部分服务器不支持，需手动更改其镜像版本为 4.4.24**
......
docSite/content/docs/pricing.md
...@@ -30,6 +30,10 @@ weight: 10 ...@@ -30,6 +30,10 @@ weight: 10
| 星火2.0 - 对话 | 0.01 | | 星火2.0 - 对话 | 0.01 |
| chatglm_pro - 对话 | 0.01 | | chatglm_pro - 对话 | 0.01 |
| 通义千问 - 对话 | 0.01 | | 通义千问 - 对话 | 0.01 |
| 问题分类 | 0.03 |
| 内容提取 | 0.03 |
| 下一步指引 | 0.015 |
{{< /table >}} {{< /table >}}
{{% alert context="warning" %}} {{% alert context="warning" %}}
......
files/deploy/fastgpt/docker-compose.yml
...@@ -2,8 +2,8 @@ ...@@ -2,8 +2,8 @@
version: '3.3' version: '3.3'
services: services:
pg: pg:
image: ankane/pgvector:v0.4.2 # git image: ankane/pgvector:v0.5.0 # git
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/pgvector:v0.4.2 # 阿里云 # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/pgvector:v0.5.0 # 阿里云
container_name: pg container_name: pg
restart: always restart: always
ports: # 生产环境建议不要暴露 ports: # 生产环境建议不要暴露
...@@ -66,8 +66,8 @@ networks: ...@@ -66,8 +66,8 @@ networks:
# version: '3.3' # version: '3.3'
# services: # services:
# pg: # pg:
# image: ankane/pgvector:v0.4.2 # dockerhub # image: ankane/pgvector:v0.5.0 # dockerhub
# # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/pgvector:v0.4.2 # 阿里云 # # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/pgvector:v0.5.0 # 阿里云
# container_name: pg # container_name: pg
# restart: always # restart: always
# ports: # 生产环境建议不要暴露 # ports: # 生产环境建议不要暴露
......
package.json
...@@ -8,11 +8,11 @@ ...@@ -8,11 +8,11 @@
}, },
"devDependencies": { "devDependencies": {
"husky": "^8.0.3", "husky": "^8.0.3",
"i18next": "^23.2.11",
"lint-staged": "^13.2.1", "lint-staged": "^13.2.1",
"next-i18next": "^14.0.0",
"prettier": "^3.0.3", "prettier": "^3.0.3",
"react-i18next": "^13.0.2" "i18next": "^23.2.11",
"react-i18next": "^13.0.2",
"next-i18next": "^14.0.0"
}, },
"lint-staged": { "lint-staged": {
"./**/**/*.{ts,tsx,scss}": "npm run format" "./**/**/*.{ts,tsx,scss}": "npm run format"
......
packages/common/mongo/index.ts 0 → 100644
import mongoose from 'mongoose';
export default mongoose;
export * from 'mongoose';
export const connectionMongo = global.mongodb || mongoose;
packages/common/mongo/init.ts 0 → 100644
import mongoose from './index';
import 'winston-mongodb';
import { createLogger, format, transports } from 'winston';
/**
* connect MongoDB and init data
*/
export async function connectMongo({
beforeHook,
afterHook
}: {
beforeHook?: () => any;
afterHook?: () => any;
}): Promise<void> {
if (global.mongodb) {
return;
}
global.mongodb = mongoose;
beforeHook && (await beforeHook());
// logger
initLogger();
console.log('mongo start connect');
try {
mongoose.set('strictQuery', true);
await mongoose.connect(process.env.MONGODB_URI as string, {
bufferCommands: true,
maxConnecting: Number(process.env.DB_MAX_LINK || 5),
maxPoolSize: Number(process.env.DB_MAX_LINK || 5),
minPoolSize: 2
});
console.log('mongo connected');
afterHook && (await afterHook());
} catch (error) {
console.log('error->', 'mongo connect error', error);
global.mongodb = undefined;
}
}
function initLogger() {
global.logger = createLogger({
transports: [
new transports.MongoDB({
db: process.env.MONGODB_URI as string,
collection: 'server_logs',
options: {
useUnifiedTopology: true
},
cappedSize: 500000000,
tryReconnect: true,
metaKey: 'meta',
format: format.combine(format.timestamp(), format.json())
}),
new transports.Console({
format: format.combine(
format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
format.printf((info) => {
if (info.level === 'error') {
console.log(info.meta);
return `[${info.level.toLocaleUpperCase()}]: ${[info.timestamp]}: ${info.message}`;
}
return `[${info.level.toLocaleUpperCase()}]: ${[info.timestamp]}: ${info.message}${
info.meta ? `: ${JSON.stringify(info.meta)}` : ''
}`;
})
)
})
]
});
}
packages/common/package.json
{ {
"name": "@fastgpt/common", "name": "@fastgpt/common",
"version": "1.0.0" "version": "1.0.0",
"dependencies": {
"mongoose": "^7.0.2",
"winston": "^3.10.0",
"winston-mongodb": "^5.1.1",
"axios": "^1.5.1"
},
"devDependencies": {
"@types/node": "^20.8.5"
}
} }
projects/app/src/service/common/api/plugins.ts → packages/common/plusApi/censor.ts
import { POST } from './request'; import { POST } from './request';
export const postTextCensor = (data: { text: string }) => export const postTextCensor = (data: { text: string }) =>
POST<{ code?: number; message: string }>('/plugins/censor/text_baidu', data) POST<{ code?: number; message: string }>('/common/censor/text_baidu', data)
.then((res) => { .then((res) => {
if (res?.code === 5000) { if (res?.code === 5000) {
return Promise.reject(res); return Promise.reject(res);
......
packages/common/tools/str.ts
import crypto from 'crypto';
export function strIsLink(str?: string) { export function strIsLink(str?: string) {
if (!str) return false; if (!str) return false;
if (/^((http|https)?:\/\/|www\.|\/)[^\s/$.?#].[^\s]*$/i.test(str)) return true; if (/^((http|https)?:\/\/|www\.|\/)[^\s/$.?#].[^\s]*$/i.test(str)) return true;
return false; return false;
} }
export const hashStr = (psw: string) => {
return crypto.createHash('sha256').update(psw).digest('hex');
};
/* simple text, remove chinese space and extra \n */
export const simpleText = (text: string) => {
text = text.replace(/([\u4e00-\u9fa5])[\s&&[^\n]]+([\u4e00-\u9fa5])/g, '$1$2');
text = text.replace(/\n{2,}/g, '\n');
text = text.replace(/[\s&&[^\n]]{2,}/g, ' ');
text = text.replace(/[\x00-\x08]/g, ' ');
// replace empty \n
let newText = '';
let lastChar = '';
for (let i = 0; i < text.length; i++) {
const currentChar = text[i];
if (currentChar === '\n' && !/[。？！；.?!;]/g.test(lastChar)) {
} else {
newText += currentChar;
}
lastChar = currentChar;
}
return newText;
};
packages/common/type/index.d.ts 0 → 100644
import type { Mongoose } from '../mongo';
import type { Logger } from 'winston';
export type FeConfigsType = {
show_emptyChat?: boolean;
show_register?: boolean;
show_appStore?: boolean;
show_contact?: boolean;
show_git?: boolean;
show_doc?: boolean;
show_pay?: boolean;
show_openai_account?: boolean;
show_promotion?: boolean;
hide_app_flow?: boolean;
openAPIUrl?: string;
systemTitle?: string;
authorText?: string;
googleClientVerKey?: string;
isPlus?: boolean;
oauth?: {
github?: string;
google?: string;
};
limit?: {
exportLimitMinutes?: number;
};
scripts?: { [key: string]: string }[];
};
export type SystemEnvType = {
pluginBaseUrl?: string;
openapiPrefix?: string;
vectorMaxProcess: number;
qaMaxProcess: number;
pgHNSWEfSearch: number;
};
declare global {
var mongodb: Mongoose | undefined;
var logger: Logger;
var feConfigs: FeConfigsType;
var systemEnv: SystemEnvType;
}
packages/core/ai/config.ts
import { UserModelSchema } from '../user/type'; import type { UserModelSchema } from '@fastgpt/support/user/type.d';
import OpenAI from 'openai'; import OpenAI from 'openai';
export const openaiBaseUrl = process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1'; export const openaiBaseUrl = process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1';
......
packages/core/ai/functions/createQuestionGuide.ts
...@@ -10,7 +10,7 @@ export async function createQuestionGuide({ ...@@ -10,7 +10,7 @@ export async function createQuestionGuide({
messages: ChatCompletionRequestMessage[]; messages: ChatCompletionRequestMessage[];
model: string; model: string;
}) { }) {
const ai = getAIApi(); const ai = getAIApi(undefined, 48000);
const data = await ai.chat.completions.create({ const data = await ai.chat.completions.create({
model: model, model: model,
temperature: 0, temperature: 0,
...@@ -25,7 +25,7 @@ export async function createQuestionGuide({ ...@@ -25,7 +25,7 @@ export async function createQuestionGuide({
stream: false stream: false
}); });
const answer = data.choices?.[0].message?.content || ''; const answer = data.choices?.[0]?.message?.content || '';
const totalTokens = data.usage?.total_tokens || 0; const totalTokens = data.usage?.total_tokens || 0;
const start = answer.indexOf('['); const start = answer.indexOf('[');
......
packages/core/dataset/constant.ts
export enum DatasetTypeEnum {
folder = 'folder',
dataset = 'dataset'
}
export const DatasetTypeMap = {
[DatasetTypeEnum.folder]: {
name: 'folder'
},
[DatasetTypeEnum.dataset]: {
name: 'dataset'
}
};
export enum FileStatusEnum {
embedding = 'embedding',
ready = 'ready'
}
export enum DatasetSpecialIdEnum { export enum DatasetSpecialIdEnum {
manual = 'manual', manual = 'manual',
mark = 'mark' mark = 'mark'
...@@ -13,3 +32,5 @@ export const datasetSpecialIdMap = { ...@@ -13,3 +32,5 @@ export const datasetSpecialIdMap = {
} }
}; };
export const datasetSpecialIds: string[] = [DatasetSpecialIdEnum.manual, DatasetSpecialIdEnum.mark]; export const datasetSpecialIds: string[] = [DatasetSpecialIdEnum.manual, DatasetSpecialIdEnum.mark];
export const FolderAvatarSrc = '/imgs/files/folder.svg';
projects/app/src/service/models/kb.ts → packages/core/dataset/schema.ts
import { Schema, model, models, Model } from 'mongoose'; import { connectionMongo, type Model } from '@fastgpt/common/mongo';
import { kbSchema as SchemaType } from '@/types/mongoSchema'; const { Schema, model, models } = connectionMongo;
import { KbTypeMap } from '@/constants/dataset'; import { DatasetSchemaType } from './type';
import { DatasetTypeMap } from './constant';
const kbSchema = new Schema({ const DatasetSchema = new Schema({
parentId: { parentId: {
type: Schema.Types.ObjectId, type: Schema.Types.ObjectId,
ref: 'kb', ref: 'kb',
...@@ -32,7 +33,7 @@ const kbSchema = new Schema({ ...@@ -32,7 +33,7 @@ const kbSchema = new Schema({
}, },
type: { type: {
type: String, type: String,
enum: Object.keys(KbTypeMap), enum: Object.keys(DatasetTypeMap),
required: true, required: true,
default: 'dataset' default: 'dataset'
}, },
...@@ -42,4 +43,4 @@ const kbSchema = new Schema({ ...@@ -42,4 +43,4 @@ const kbSchema = new Schema({
} }
}); });
export const KB: Model<SchemaType> = models['kb'] || model('kb', kbSchema); export const MongoDataset: Model<DatasetSchemaType> = models['kb'] || model('kb', DatasetSchema);
packages/core/dataset/type.d.ts 0 → 100644
import { DatasetTypeEnum } from './constant';
export type DatasetSchemaType = {
_id: string;
userId: string;
parentId: string;
updateTime: Date;
avatar: string;
name: string;
vectorModel: string;
tags: string[];
type: `${DatasetTypeEnum}`;
};
packages/core/tsconfig.json
...@@ -16,6 +16,6 @@ ...@@ -16,6 +16,6 @@
"incremental": true, "incremental": true,
"baseUrl": "." "baseUrl": "."
}, },
"include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", "**/*.d.ts"], "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", "**/*.d.ts", "../**/*.d.ts"],
"exclude": ["node_modules"] "exclude": ["node_modules"]
} }
projects/app/src/service/support/openapi/auth.ts → packages/support/openapi/auth.ts
import { ERROR_ENUM } from '@/service/errorCode'; import { ERROR_ENUM } from '@fastgpt/common/constant/errorCode';
import { updateApiKeyUsedTime } from './index'; import { updateApiKeyUsedTime } from './tools';
import { OpenApi } from './schema'; import { MongoOpenApi } from './schema';
import { POST } from '@fastgpt/common/plusApi/request';
import { OpenApiSchema } from './type.d';
export type AuthOpenApiLimitProps = { openApi: OpenApiSchema };
export async function authOpenApiKey({ apikey }: { apikey: string }) { export async function authOpenApiKey({ apikey }: { apikey: string }) {
if (!apikey) { if (!apikey) {
...@@ -8,7 +12,7 @@ export async function authOpenApiKey({ apikey }: { apikey: string }) { ...@@ -8,7 +12,7 @@ export async function authOpenApiKey({ apikey }: { apikey: string }) {
} }
try { try {
const openApi = await OpenApi.findOne({ apiKey: apikey }); const openApi = await MongoOpenApi.findOne({ apiKey: apikey });
if (!openApi) { if (!openApi) {
return Promise.reject(ERROR_ENUM.unAuthApiKey); return Promise.reject(ERROR_ENUM.unAuthApiKey);
} }
...@@ -16,17 +20,7 @@ export async function authOpenApiKey({ apikey }: { apikey: string }) { ...@@ -16,17 +20,7 @@ export async function authOpenApiKey({ apikey }: { apikey: string }) {
// auth limit // auth limit
if (global.feConfigs?.isPlus) { if (global.feConfigs?.isPlus) {
if (openApi?.limit?.expiredTime && openApi.limit.expiredTime.getTime() < Date.now()) { await POST('/support/openapi/authLimit', { openApi } as AuthOpenApiLimitProps);
return Promise.reject(`Key ${openApi.apiKey} is expired`);
}
if (
openApi?.limit?.credit &&
openApi.limit.credit > -1 &&
openApi.usage > openApi.limit.credit
) {
return Promise.reject(`Key ${openApi.apiKey} is over usage`);
}
} }
updateApiKeyUsedTime(openApi._id); updateApiKeyUsedTime(openApi._id);
......
projects/app/src/service/support/openapi/schema.ts → packages/support/openapi/schema.ts
import { Schema, model, models, Model } from 'mongoose'; import { connectionMongo, type Model } from '@fastgpt/common/mongo';
import { OpenApiSchema } from '@/types/support/openapi'; const { Schema, model, models } = connectionMongo;
import { OpenApiSchema } from './type.d';
import { PRICE_SCALE } from '@fastgpt/common/bill/constants'; import { PRICE_SCALE } from '@fastgpt/common/bill/constants';
import { formatPrice } from '@fastgpt/common/bill/index'; import { formatPrice } from '@fastgpt/common/bill/index';
...@@ -54,4 +55,5 @@ const OpenApiSchema = new Schema( ...@@ -54,4 +55,5 @@ const OpenApiSchema = new Schema(
} }
); );
export const OpenApi: Model<OpenApiSchema> = models['openapi'] || model('openapi', OpenApiSchema); export const MongoOpenApi: Model<OpenApiSchema> =
models['openapi'] || model('openapi', OpenApiSchema);
projects/app/src/service/support/openapi/index.ts → packages/support/openapi/tools.ts
import { OpenApi } from './schema'; import { MongoOpenApi } from './schema';
export async function updateApiKeyUsedTime(id: string) { export async function updateApiKeyUsedTime(id: string) {
await OpenApi.findByIdAndUpdate(id, { await MongoOpenApi.findByIdAndUpdate(id, {
lastUsedTime: new Date() lastUsedTime: new Date()
}); });
} }
export async function updateApiKeyUsage({ apikey, usage }: { apikey: string; usage: number }) { export async function updateApiKeyUsage({ apikey, usage }: { apikey: string; usage: number }) {
await OpenApi.findOneAndUpdate( await MongoOpenApi.findOneAndUpdate(
{ apiKey: apikey }, { apiKey: apikey },
{ {
$inc: { $inc: {
......
packages/support/outLink/auth.ts 0 → 100644
import { AuthUserTypeEnum, authBalanceByUid } from '../user/auth';
import { MongoOutLink } from './schema';
import { POST } from '@fastgpt/common/plusApi/request';
import { OutLinkSchema } from './type.d';
export type AuthLinkProps = { ip?: string | null; authToken?: string; question: string };
export type AuthLinkLimitProps = AuthLinkProps & { outLink: OutLinkSchema };
export async function authOutLinkChat({
shareId,
ip,
authToken,
question
}: AuthLinkProps & {
shareId: string;
}) {
// get outLink
const outLink = await MongoOutLink.findOne({
shareId
});
if (!outLink) {
return Promise.reject('分享链接无效');
}
const uid = String(outLink.userId);
const [user] = await Promise.all([
authBalanceByUid(uid), // authBalance
...(global.feConfigs?.isPlus ? [authOutLinkLimit({ outLink, ip, authToken, question })] : []) // limit auth
]);
return {
user,
userId: String(outLink.userId),
appId: String(outLink.appId),
authType: AuthUserTypeEnum.token,
responseDetail: outLink.responseDetail
};
}
export function authOutLinkLimit(data: AuthLinkLimitProps) {
return POST('/support/outLink/authLimit', data);
}
export async function authOutLinkId({ id }: { id: string }) {
const outLink = await MongoOutLink.findOne({
shareId: id
});
if (!outLink) {
return Promise.reject('分享链接无效');
}
return {
userId: String(outLink.userId)
};
}
export type AuthShareChatInitProps = {
authToken?: string;
tokenUrl?: string;
};
export function authShareChatInit(data: AuthShareChatInitProps) {
return POST('/support/outLink/authShareChatInit', data);
}
packages/support/outLink/constant.ts 0 → 100644
export enum OutLinkTypeEnum {
'share' = 'share',
'iframe' = 'iframe',
apikey = 'apikey'
}
projects/app/src/service/support/outLink/schema.ts → packages/support/outLink/schema.ts
import { Schema, model, models, Model } from 'mongoose'; import { connectionMongo, type Model } from '@fastgpt/common/mongo';
import { OutLinkSchema as SchemaType } from '@/types/support/outLink'; const { Schema, model, models } = connectionMongo;
import { OutLinkTypeEnum } from '@/constants/chat'; import { OutLinkSchema as SchemaType } from './type.d';
import { OutLinkTypeEnum } from './constant';
const OutLinkSchema = new Schema({ const OutLinkSchema = new Schema({
shareId: { shareId: {
...@@ -55,4 +56,5 @@ const OutLinkSchema = new Schema({ ...@@ -55,4 +56,5 @@ const OutLinkSchema = new Schema({
} }
}); });
export const OutLink: Model<SchemaType> = models['outlinks'] || model('outlinks', OutLinkSchema); export const MongoOutLink: Model<SchemaType> =
models['outlinks'] || model('outlinks', OutLinkSchema);
projects/app/src/service/support/outLink/index.ts → packages/support/outLink/tools.ts
import { addLog } from '@/service/utils/tools';
import { ChatHistoryItemResType } from '@/types/chat';
import axios from 'axios'; import axios from 'axios';
import { OutLink } from './schema'; import { MongoOutLink } from './schema';
export const updateOutLinkUsage = async ({ export const updateOutLinkUsage = async ({
shareId, shareId,
...@@ -11,7 +9,7 @@ export const updateOutLinkUsage = async ({ ...@@ -11,7 +9,7 @@ export const updateOutLinkUsage = async ({
total: number; total: number;
}) => { }) => {
try { try {
await OutLink.findOneAndUpdate( await MongoOutLink.findOneAndUpdate(
{ shareId }, { shareId },
{ {
$inc: { total }, $inc: { total },
...@@ -19,7 +17,7 @@ export const updateOutLinkUsage = async ({ ...@@ -19,7 +17,7 @@ export const updateOutLinkUsage = async ({
} }
); );
} catch (err) { } catch (err) {
addLog.error('update shareChat error', err); console.log('update shareChat error', err);
} }
}; };
...@@ -30,11 +28,11 @@ export const pushResult2Remote = async ({ ...@@ -30,11 +28,11 @@ export const pushResult2Remote = async ({
}: { }: {
authToken?: string; authToken?: string;
shareId?: string; shareId?: string;
responseData?: ChatHistoryItemResType[]; responseData?: any[];
}) => { }) => {
if (!shareId || !authToken) return; if (!shareId || !authToken) return;
try { try {
const outLink = await OutLink.findOne({ const outLink = await MongoOutLink.findOne({
shareId shareId
}); });
if (!outLink?.limit?.hookUrl) return; if (!outLink?.limit?.hookUrl) return;
......
projects/app/src/types/support/outLink.d.ts → packages/support/outLink/type.d.ts
import { OutLinkTypeEnum } from '@/constants/chat'; import { OutLinkTypeEnum } from './constant';
export interface OutLinkSchema { export type OutLinkSchema = {
_id: string; _id: string;
shareId: string; shareId: string;
userId: string; userId: string;
...@@ -16,7 +16,7 @@ export interface OutLinkSchema { ...@@ -16,7 +16,7 @@ export interface OutLinkSchema {
credit: number; credit: number;
hookUrl?: string; hookUrl?: string;
}; };
} };
export type OutLinkEditType = { export type OutLinkEditType = {
_id?: string; _id?: string;
......
packages/support/package.json
...@@ -2,6 +2,13 @@ ...@@ -2,6 +2,13 @@
"name": "@fastgpt/support", "name": "@fastgpt/support",
"version": "1.0.0", "version": "1.0.0",
"dependencies": { "dependencies": {
"@fastgpt/common": "workspace:*" "@fastgpt/common": "workspace:*",
"cookie": "^0.5.0",
"jsonwebtoken": "^9.0.2",
"axios": "^1.5.1"
},
"devDependencies": {
"@types/cookie": "^0.5.2",
"@types/jsonwebtoken": "^9.0.3"
} }
} }
packages/support/tsconfig.json
...@@ -14,8 +14,11 @@ ...@@ -14,8 +14,11 @@
"isolatedModules": true, "isolatedModules": true,
"jsx": "preserve", "jsx": "preserve",
"incremental": true, "incremental": true,
"baseUrl": "." "baseUrl": ".",
"paths": {
"@/*": ["./*"]
}
}, },
"include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", "**/*.d.ts"], "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", "**/*.d.ts", "../**/*.d.ts"],
"exclude": ["node_modules"] "exclude": ["node_modules"]
} }
packages/support/user/auth.ts 0 → 100644
import Cookie from 'cookie';
import { authJWT } from './tools';
import { authOpenApiKey } from '../openapi/auth';
import { authOutLinkId } from '../outLink/auth';
import { MongoUser } from './schema';
import type { UserModelSchema } from './type.d';
import { ERROR_ENUM } from '@fastgpt/common/constant/errorCode';
export enum AuthUserTypeEnum {
token = 'token',
root = 'root',
apikey = 'apikey',
outLink = 'outLink'
}
/* auth balance */
export const authBalanceByUid = async (uid: string) => {
const user = await MongoUser.findById<UserModelSchema>(
uid,
'_id username balance openaiAccount timezone'
);
if (!user) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
if (user.balance <= 0) {
return Promise.reject(ERROR_ENUM.insufficientQuota);
}
return user;
};
/* uniform auth user */
export const authUser = async ({
req,
authToken = false,
authRoot = false,
authApiKey = false,
authBalance = false,
authOutLink
}: {
req: any;
authToken?: boolean;
authRoot?: boolean;
authApiKey?: boolean;
authBalance?: boolean;
authOutLink?: boolean;
}) => {
const authCookieToken = async (cookie?: string, token?: string): Promise<string> => {
// 获取 cookie
const cookies = Cookie.parse(cookie || '');
const cookieToken = cookies.token || token;
if (!cookieToken) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
return await authJWT(cookieToken);
};
// from authorization get apikey
const parseAuthorization = async (authorization?: string) => {
if (!authorization) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
// Bearer fastgpt-xxxx-appId
const auth = authorization.split(' ')[1];
if (!auth) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
const { apikey, appId: authorizationAppid = '' } = await (async () => {
const arr = auth.split('-');
// abandon
if (arr.length === 3) {
return {
apikey: `${arr[0]}-${arr[1]}`,
appId: arr[2]
};
}
if (arr.length === 2) {
return {
apikey: auth
};
}
return Promise.reject(ERROR_ENUM.unAuthorization);
})();
// auth apikey
const { userId, appId: apiKeyAppId = '' } = await authOpenApiKey({ apikey });
return {
uid: userId,
apikey,
appId: apiKeyAppId || authorizationAppid
};
};
// root user
const parseRootKey = async (rootKey?: string, userId = '') => {
if (!rootKey || !process.env.ROOT_KEY || rootKey !== process.env.ROOT_KEY) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
return userId;
};
const { cookie, token, apikey, rootkey, userid, authorization } = (req.headers || {}) as {
cookie?: string;
token?: string;
apikey?: string;
rootkey?: string; // abandon
userid?: string;
authorization?: string;
};
const { shareId } = (req?.body || {}) as { shareId?: string };
let uid = '';
let appId = '';
let openApiKey = apikey;
let authType: `${AuthUserTypeEnum}` = AuthUserTypeEnum.token;
if (authOutLink && shareId) {
const res = await authOutLinkId({ id: shareId });
uid = res.userId;
authType = AuthUserTypeEnum.outLink;
} else if (authToken && (cookie || token)) {
// user token(from fastgpt web)
uid = await authCookieToken(cookie, token);
authType = AuthUserTypeEnum.token;
} else if (authRoot && rootkey) {
// root user
uid = await parseRootKey(rootkey, userid);
authType = AuthUserTypeEnum.root;
} else if (authApiKey && apikey) {
// apikey
const parseResult = await authOpenApiKey({ apikey });
uid = parseResult.userId;
authType = AuthUserTypeEnum.apikey;
openApiKey = parseResult.apikey;
} else if (authApiKey && authorization) {
// apikey from authorization
const authResponse = await parseAuthorization(authorization);
uid = authResponse.uid;
appId = authResponse.appId;
openApiKey = authResponse.apikey;
authType = AuthUserTypeEnum.apikey;
}
// not rootUser and no uid, reject request
if (!rootkey && !uid) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
// balance check
const user = await (() => {
if (authBalance) {
return authBalanceByUid(uid);
}
})();
return {
userId: String(uid),
appId,
authType,
user,
apikey: openApiKey
};
};
projects/app/src/service/models/user.ts → packages/support/user/schema.ts
import { Schema, model, models, Model } from 'mongoose'; import { connectionMongo, type Model } from '@fastgpt/common/mongo';
import { hashPassword } from '@/service/utils/tools'; const { Schema, model, models } = connectionMongo;
import { hashStr } from '@fastgpt/common/tools/str';
import { PRICE_SCALE } from '@fastgpt/common/bill/constants'; import { PRICE_SCALE } from '@fastgpt/common/bill/constants';
import { UserModelSchema } from '@/types/mongoSchema'; import type { UserModelSchema } from './type.d';
const UserSchema = new Schema({ const UserSchema = new Schema({
username: { username: {
...@@ -13,8 +14,8 @@ const UserSchema = new Schema({ ...@@ -13,8 +14,8 @@ const UserSchema = new Schema({
password: { password: {
type: String, type: String,
required: true, required: true,
set: (val: string) => hashPassword(val), set: (val: string) => hashStr(val),
get: (val: string) => hashPassword(val), get: (val: string) => hashStr(val),
select: false select: false
}, },
createTime: { createTime: {
...@@ -42,6 +43,9 @@ const UserSchema = new Schema({ ...@@ -42,6 +43,9 @@ const UserSchema = new Schema({
exportKbTime: { exportKbTime: {
// Every half hour // Every half hour
type: Date type: Date
},
datasetMaxCount: {
type: Number
} }
}, },
openaiAccount: { openaiAccount: {
...@@ -56,4 +60,4 @@ const UserSchema = new Schema({ ...@@ -56,4 +60,4 @@ const UserSchema = new Schema({
} }
}); });
export const User: Model<UserModelSchema> = models['user'] || model('user', UserSchema); export const MongoUser: Model<UserModelSchema> = models['user'] || model('user', UserSchema);
packages/support/user/tools.ts 0 → 100644
import jwt from 'jsonwebtoken';
import { ERROR_ENUM } from '@fastgpt/common/constant/errorCode';
/* 生成 token */
export const generateToken = (userId: string) => {
const key = process.env.TOKEN_KEY as string;
const token = jwt.sign(
{
userId,
exp: Math.floor(Date.now() / 1000) + 60 * 60 * 24 * 7
},
key
);
return token;
};
// auth token
export const authJWT = (token: string) =>
new Promise<string>((resolve, reject) => {
const key = process.env.TOKEN_KEY as string;
jwt.verify(token, key, function (err, decoded: any) {
if (err || !decoded?.userId) {
reject(ERROR_ENUM.unAuthorization);
return;
}
resolve(decoded.userId);
});
});
packages/core/user/type.d.ts → packages/support/user/type.d.ts
...@@ -15,5 +15,6 @@ export type UserModelSchema = { ...@@ -15,5 +15,6 @@ export type UserModelSchema = {
}; };
limit: { limit: {
exportKbTime?: Date; exportKbTime?: Date;
datasetMaxCount?: number;
}; };
}; };
pnpm-lock.yaml
This source diff could not be displayed because it is too large. You can view the blob instead.
projects/app/data/config.json
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
"SystemParams": { "SystemParams": {
"vectorMaxProcess": 15, "vectorMaxProcess": 15,
"qaMaxProcess": 15, "qaMaxProcess": 15,
"pgIvfflatProbe": 20 "pgHNSWEfSearch ": 40
}, },
"ChatModels": [ "ChatModels": [
{ {
......
projects/app/next.config.js
...@@ -5,9 +5,8 @@ const path = require('path'); ...@@ -5,9 +5,8 @@ const path = require('path');
const nextConfig = { const nextConfig = {
i18n, i18n,
output: 'standalone', output: 'standalone',
reactStrictMode: false, reactStrictMode: process.env.NODE_ENV === 'development' ? false : true,
compress: true, compress: true,
transpilePackages: ['@fastgpt/*'],
webpack(config, { isServer }) { webpack(config, { isServer }) {
if (!isServer) { if (!isServer) {
config.resolve = { config.resolve = {
...@@ -18,6 +17,16 @@ const nextConfig = { ...@@ -18,6 +17,16 @@ const nextConfig = {
} }
}; };
} }
Object.assign(config.resolve.alias, {
'@mongodb-js/zstd': false,
'@aws-sdk/credential-providers': false,
snappy: false,
aws4: false,
'mongodb-client-encryption': false,
kerberos: false,
'supports-color': false,
'bson-ext': false
});
config.module = { config.module = {
...config.module, ...config.module,
rules: config.module.rules.concat([ rules: config.module.rules.concat([
...@@ -27,12 +36,15 @@ const nextConfig = { ...@@ -27,12 +36,15 @@ const nextConfig = {
use: ['@svgr/webpack'] use: ['@svgr/webpack']
} }
]), ]),
exprContextCritical: false exprContextCritical: false,
unknownContextCritical: false
}; };
return config; return config;
}, },
transpilePackages: ['@fastgpt/*'],
experimental: { experimental: {
serverComponentsExternalPackages: ['mongoose'],
outputFileTracingRoot: path.join(__dirname, '../../') outputFileTracingRoot: path.join(__dirname, '../../')
} }
}; };
......
projects/app/package.json
...@@ -22,9 +22,7 @@ ...@@ -22,9 +22,7 @@
"@mozilla/readability": "^0.4.4", "@mozilla/readability": "^0.4.4",
"@tanstack/react-query": "^4.24.10", "@tanstack/react-query": "^4.24.10",
"@types/nprogress": "^0.2.0", "@types/nprogress": "^0.2.0",
"axios": "^1.3.3", "axios": "^1.5.1",
"cookie": "^0.5.0",
"crypto": "^1.0.1",
"date-fns": "^2.30.0", "date-fns": "^2.30.0",
"dayjs": "^1.11.7", "dayjs": "^1.11.7",
"downloadjs": "^1.4.7", "downloadjs": "^1.4.7",
...@@ -33,21 +31,18 @@ ...@@ -33,21 +31,18 @@
"formidable": "^2.1.1", "formidable": "^2.1.1",
"framer-motion": "^9.0.6", "framer-motion": "^9.0.6",
"hyperdown": "^2.4.29", "hyperdown": "^2.4.29",
"i18next": "^22.5.1",
"immer": "^9.0.19", "immer": "^9.0.19",
"js-cookie": "^3.0.5", "js-cookie": "^3.0.5",
"js-tiktoken": "^1.0.7", "js-tiktoken": "^1.0.7",
"jschardet": "^3.0.0", "jschardet": "^3.0.0",
"jsdom": "^22.1.0", "jsdom": "^22.1.0",
"jsonwebtoken": "^9.0.0", "jsonwebtoken": "^9.0.2",
"lodash": "^4.17.21", "lodash": "^4.17.21",
"mammoth": "^1.5.1", "mammoth": "^1.5.1",
"mermaid": "^10.2.3", "mermaid": "^10.2.3",
"mongoose": "^6.10.0",
"multer": "1.4.5-lts.1", "multer": "1.4.5-lts.1",
"nanoid": "^4.0.1", "nanoid": "^4.0.1",
"next": "13.5.2", "next": "13.5.2",
"next-i18next": "^14.0.3",
"nextjs-cors": "^2.1.2", "nextjs-cors": "^2.1.2",
"nprogress": "^0.2.0", "nprogress": "^0.2.0",
"papaparse": "^5.4.1", "papaparse": "^5.4.1",
...@@ -57,7 +52,6 @@ ...@@ -57,7 +52,6 @@
"react-day-picker": "^8.7.1", "react-day-picker": "^8.7.1",
"react-dom": "18.2.0", "react-dom": "18.2.0",
"react-hook-form": "^7.43.1", "react-hook-form": "^7.43.1",
"react-i18next": "^12.3.1",
"react-markdown": "^8.0.7", "react-markdown": "^8.0.7",
"react-syntax-highlighter": "^15.5.0", "react-syntax-highlighter": "^15.5.0",
"reactflow": "^11.7.4", "reactflow": "^11.7.4",
...@@ -68,21 +62,20 @@ ...@@ -68,21 +62,20 @@
"request-ip": "^3.3.0", "request-ip": "^3.3.0",
"sass": "^1.58.3", "sass": "^1.58.3",
"timezones-list": "^3.0.2", "timezones-list": "^3.0.2",
"winston": "^3.10.0", "zustand": "^4.3.5",
"winston-mongodb": "^5.1.1", "i18next": "^23.2.11",
"zustand": "^4.3.5" "react-i18next": "^13.0.2",
"next-i18next": "^14.0.0"
}, },
"devDependencies": { "devDependencies": {
"@svgr/webpack": "^6.5.1", "@svgr/webpack": "^6.5.1",
"@types/cookie": "^0.5.1",
"@types/downloadjs": "^1.4.3", "@types/downloadjs": "^1.4.3",
"@types/formidable": "^2.0.5", "@types/formidable": "^2.0.5",
"@types/js-cookie": "^3.0.3", "@types/js-cookie": "^3.0.3",
"@types/jsdom": "^21.1.1", "@types/jsdom": "^21.1.1",
"@types/jsonwebtoken": "^9.0.1", "@types/jsonwebtoken": "^9.0.3",
"@types/lodash": "^4.14.191", "@types/lodash": "^4.14.191",
"@types/multer": "^1.4.7", "@types/multer": "^1.4.7",
"@types/node": "18.14.0",
"@types/papaparse": "^5.3.7", "@types/papaparse": "^5.3.7",
"@types/pg": "^8.6.6", "@types/pg": "^8.6.6",
"@types/react": "18.0.28", "@types/react": "18.0.28",
......
projects/app/pnpm-lock.yaml deleted 100644 → 0
This source diff could not be displayed because it is too large. You can view the blob instead.
projects/app/public/locales/en/common.json
...@@ -181,6 +181,7 @@ ...@@ -181,6 +181,7 @@
"Choice QA Desc": "The index is constructed with the structure of QA pairs, and ADAPTS to various scenarios such as Q&A and reading", "Choice QA Desc": "The index is constructed with the structure of QA pairs, and ADAPTS to various scenarios such as Q&A and reading",
"Choice Visual": "Visual workflow", "Choice Visual": "Visual workflow",
"Choice Visual Desc": "Visualize modular operations, easily implement complex workflows, and make your AI no longer monolithic", "Choice Visual Desc": "Visualize modular operations, easily implement complex workflows, and make your AI no longer monolithic",
"Commercial": "Commercial",
"Community": "Community", "Community": "Community",
"Dateset": "", "Dateset": "",
"Dateset Desc": "", "Dateset Desc": "",
......
projects/app/public/locales/zh/common.json
...@@ -181,6 +181,7 @@ ...@@ -181,6 +181,7 @@
"Choice QA Desc": "采用 QA 对的结构构建索引，适应问答、阅读等多种场景", "Choice QA Desc": "采用 QA 对的结构构建索引，适应问答、阅读等多种场景",
"Choice Visual": "可视化工作流", "Choice Visual": "可视化工作流",
"Choice Visual Desc": "可视化模块操作，轻松实现复杂工作流，让你的 AI 不再单一", "Choice Visual Desc": "可视化模块操作，轻松实现复杂工作流，让你的 AI 不再单一",
"Commercial": "商务咨询",
"Community": "社区", "Community": "社区",
"Dateset": "自动数据预处理", "Dateset": "自动数据预处理",
"Dateset Desc": "提供手动输入、直接分段、LLM 自动处理和 CSV 等多种数据导入途径", "Dateset Desc": "提供手动输入、直接分段、LLM 自动处理和 CSV 等多种数据导入途径",
......
projects/app/src/components/ChatBox/SelectDataset.tsx
...@@ -4,7 +4,7 @@ import { useTranslation } from 'next-i18next'; ...@@ -4,7 +4,7 @@ import { useTranslation } from 'next-i18next';
import { useToast } from '@/web/common/hooks/useToast'; import { useToast } from '@/web/common/hooks/useToast';
import Avatar from '../Avatar'; import Avatar from '../Avatar';
import MyIcon from '@/components/Icon'; import MyIcon from '@/components/Icon';
import { KbTypeEnum } from '@/constants/dataset'; import { DatasetTypeEnum } from '@fastgpt/core/dataset/constant';
import DatasetSelectModal, { useDatasetSelect } from '@/components/core/dataset/SelectModal'; import DatasetSelectModal, { useDatasetSelect } from '@/components/core/dataset/SelectModal';
const SelectDataset = ({ const SelectDataset = ({
...@@ -57,7 +57,7 @@ const SelectDataset = ({ ...@@ -57,7 +57,7 @@ const SelectDataset = ({
} }
: {})} : {})}
onClick={() => { onClick={() => {
if (item.type === KbTypeEnum.folder) { if (item.type === DatasetTypeEnum.folder) {
setParentId(item._id); setParentId(item._id);
} else { } else {
setSelectedId(item._id); setSelectedId(item._id);
......
projects/app/src/components/MyModal/index.tsx
...@@ -45,8 +45,8 @@ const MyModal = ({ ...@@ -45,8 +45,8 @@ const MyModal = ({
{...props} {...props}
> >
{!!title && <ModalHeader>{title}</ModalHeader>} {!!title && <ModalHeader>{title}</ModalHeader>}
<Box overflow={'overlay'} h={'100%'}>
{onClose && <ModalCloseButton />} {onClose && <ModalCloseButton />}
<Box overflow={'overlay'} h={'100%'}>
{children} {children}
</Box> </Box>
</ModalContent> </ModalContent>
......
projects/app/src/constants/chat.ts
...@@ -53,11 +53,5 @@ export const ChatSourceMap = { ...@@ -53,11 +53,5 @@ export const ChatSourceMap = {
} }
}; };
export enum OutLinkTypeEnum {
'share' = 'share',
'iframe' = 'iframe',
apikey = 'apikey'
}
export const HUMAN_ICON = `/icon/human.svg`; export const HUMAN_ICON = `/icon/human.svg`;
export const LOGO_ICON = `/icon/logo.svg`; export const LOGO_ICON = `/icon/logo.svg`;
projects/app/src/constants/dataset.ts
...@@ -14,23 +14,3 @@ export const defaultKbDetail: DatasetItemType = { ...@@ -14,23 +14,3 @@ export const defaultKbDetail: DatasetItemType = {
maxToken: 3000 maxToken: 3000
} }
}; };
export enum KbTypeEnum {
folder = 'folder',
dataset = 'dataset'
}
export enum FileStatusEnum {
embedding = 'embedding',
ready = 'ready'
}
export const KbTypeMap = {
[KbTypeEnum.folder]: {
name: 'folder'
},
[KbTypeEnum.dataset]: {
name: 'dataset'
}
};
export const FolderAvatarSrc = '/imgs/files/folder.svg';
projects/app/src/constants/flow/index.ts
...@@ -2,7 +2,7 @@ import type { BoxProps } from '@chakra-ui/react'; ...@@ -2,7 +2,7 @@ import type { BoxProps } from '@chakra-ui/react';
export enum FlowInputItemTypeEnum { export enum FlowInputItemTypeEnum {
systemInput = 'systemInput', // history, userChatInput, variableInput systemInput = 'systemInput', // history, userChatInput, variableInput
input = 'input', input = 'input', // one line input
textarea = 'textarea', textarea = 'textarea',
numberInput = 'numberInput', numberInput = 'numberInput',
select = 'select', select = 'select',
...@@ -10,6 +10,7 @@ export enum FlowInputItemTypeEnum { ...@@ -10,6 +10,7 @@ export enum FlowInputItemTypeEnum {
custom = 'custom', custom = 'custom',
target = 'target', // data input target = 'target', // data input
switch = 'switch', switch = 'switch',
chatInput = 'chatInput',
selectApp = 'selectApp', selectApp = 'selectApp',
// chat special input // chat special input
quoteList = 'quoteList', quoteList = 'quoteList',
......
projects/app/src/constants/model.ts
import type { AppSchema } from '@/types/mongoSchema'; import type { AppSchema } from '@/types/mongoSchema';
import type { OutLinkEditType } from '@/types/support/outLink'; import type { OutLinkEditType } from '@fastgpt/support/outLink/type.d';
export const defaultApp: AppSchema = { export const defaultApp: AppSchema = {
_id: '', _id: '',
......
projects/app/src/global/common/api/systemRes.d.ts
...@@ -4,11 +4,13 @@ import { ...@@ -4,11 +4,13 @@ import {
type VectorModelItemType, type VectorModelItemType,
FunctionModelItemType FunctionModelItemType
} from '@/types/model'; } from '@/types/model';
import type { FeConfigsType } from '@fastgpt/common/type/index.d';
export type InitDateResponse = { export type InitDateResponse = {
chatModels: ChatModelItemType[]; chatModels: ChatModelItemType[];
qaModel: QAModelItemType; qaModel: QAModelItemType;
vectorModels: VectorModelItemType[]; vectorModels: VectorModelItemType[];
feConfigs: FeConfigsType; feConfigs: FeConfigsType;
priceMd: string;
systemVersion: string; systemVersion: string;
}; };
projects/app/src/global/core/api/datasetReq.d.ts
import { KbTypeEnum } from '@/constants/dataset'; import { DatasetTypeEnum } from '@fastgpt/core/dataset/constant';
import type { RequestPaging } from '@/types'; import type { RequestPaging } from '@/types';
import { TrainingModeEnum } from '@/constants/plugin'; import { TrainingModeEnum } from '@/constants/plugin';
import type { SearchTestItemType } from '@/types/core/dataset'; import type { SearchTestItemType } from '@/types/core/dataset';
...@@ -18,7 +18,7 @@ export type CreateDatasetParams = { ...@@ -18,7 +18,7 @@ export type CreateDatasetParams = {
tags: string[]; tags: string[];
avatar: string; avatar: string;
vectorModel?: string; vectorModel?: string;
type: `${KbTypeEnum}`; type: `${DatasetTypeEnum}`;
}; };
export type SearchTestProps = { export type SearchTestProps = {
......
projects/app/src/global/core/api/datasetRes.d.ts
import { KbTypeEnum } from '@/constants/dataset';
import type { RequestPaging } from '@/types'; import type { RequestPaging } from '@/types';
import { TrainingModeEnum } from '@/constants/plugin'; import { TrainingModeEnum } from '@/constants/plugin';
import type { SearchTestItemType } from '@/types/core/dataset'; import type { SearchTestItemType } from '@/types/core/dataset';
......
projects/app/src/global/support/api/openapiReq.d.ts
import type { OpenApiSchema } from '@/types/support/openapi'; import type { OpenApiSchema } from '@fastgpt/support/openapi/type.d';
export type GetApiKeyProps = { export type GetApiKeyProps = {
appId?: string; appId?: string;
......
projects/app/src/pages/_app.tsx
...@@ -13,10 +13,10 @@ import { appWithTranslation, useTranslation } from 'next-i18next'; ...@@ -13,10 +13,10 @@ import { appWithTranslation, useTranslation } from 'next-i18next';
import { getLangStore, setLangStore } from '@/web/common/utils/i18n'; import { getLangStore, setLangStore } from '@/web/common/utils/i18n';
import { useRouter } from 'next/router'; import { useRouter } from 'next/router';
import { useGlobalStore } from '@/web/common/store/global'; import { useGlobalStore } from '@/web/common/store/global';
import type { FeConfigsType } from '@fastgpt/common/type/index.d';
import 'nprogress/nprogress.css'; import 'nprogress/nprogress.css';
import '@/web/styles/reset.scss'; import '@/web/styles/reset.scss';
import { FeConfigsType } from '@/types';
//Binding events. //Binding events.
Router.events.on('routeChangeStart', () => NProgress.start()); Router.events.on('routeChangeStart', () => NProgress.start());
......
projects/app/src/pages/_error.tsx
...@@ -2,8 +2,6 @@ import { useEffect } from 'react'; ...@@ -2,8 +2,6 @@ import { useEffect } from 'react';
import { useRouter } from 'next/router'; import { useRouter } from 'next/router';
import { serviceSideProps } from '@/web/common/utils/i18n'; import { serviceSideProps } from '@/web/common/utils/i18n';
import { useGlobalStore } from '@/web/common/store/global'; import { useGlobalStore } from '@/web/common/store/global';
import { addLog } from '@/service/utils/tools';
import { getErrText } from '@/utils/tools';
function Error() { function Error() {
const router = useRouter(); const router = useRouter();
...@@ -36,8 +34,6 @@ function Error() { ...@@ -36,8 +34,6 @@ function Error() {
export async function getServerSideProps(context: any) { export async function getServerSideProps(context: any) {
console.log('[render error]: ', context); console.log('[render error]: ', context);
addLog.error(getErrText(context?.res));
return { return {
props: { ...(await serviceSideProps(context)) } props: { ...(await serviceSideProps(context)) }
}; };
......
projects/app/src/pages/account/components/PayModal.tsx
...@@ -6,10 +6,9 @@ import { useQuery } from '@tanstack/react-query'; ...@@ -6,10 +6,9 @@ import { useQuery } from '@tanstack/react-query';
import { useRouter } from 'next/router'; import { useRouter } from 'next/router';
import { getErrText } from '@/utils/tools'; import { getErrText } from '@/utils/tools';
import { useTranslation } from 'react-i18next'; import { useTranslation } from 'react-i18next';
import { formatPrice } from '@fastgpt/common/bill/index';
import Markdown from '@/components/Markdown'; import Markdown from '@/components/Markdown';
import MyModal from '@/components/MyModal'; import MyModal from '@/components/MyModal';
import { vectorModelList, chatModelList, qaModel } from '@/web/common/store/static'; import { priceMd } from '@/web/common/store/static';
const PayModal = ({ onClose }: { onClose: () => void }) => { const PayModal = ({ onClose }: { onClose: () => void }) => {
const router = useRouter(); const router = useRouter();
...@@ -70,10 +69,15 @@ const PayModal = ({ onClose }: { onClose: () => void }) => { ...@@ -70,10 +69,15 @@ const PayModal = ({ onClose }: { onClose: () => void }) => {
title={t('user.Pay')} title={t('user.Pay')}
isCentered={!payId} isCentered={!payId}
> >
<ModalBody py={0}> <ModalBody
p={0}
h={payId ? 'auto' : ['auto', '70vh']}
display={'flex'}
flexDirection={'column'}
>
{!payId && ( {!payId && (
<> <>
<Grid gridTemplateColumns={'repeat(4,1fr)'} gridGap={5} mb={4}> <Grid gridTemplateColumns={'repeat(4,1fr)'} gridGap={5} mb={4} px={6}>
{[10, 20, 50, 100].map((item) => ( {[10, 20, 50, 100].map((item) => (
<Button <Button
key={item} key={item}
...@@ -84,7 +88,7 @@ const PayModal = ({ onClose }: { onClose: () => void }) => { ...@@ -84,7 +88,7 @@ const PayModal = ({ onClose }: { onClose: () => void }) => {
</Button> </Button>
))} ))}
</Grid> </Grid>
<Box mb={4}> <Box mb={4} px={6}>
<Input <Input
value={inputVal} value={inputVal}
type={'number'} type={'number'}
...@@ -95,18 +99,9 @@ const PayModal = ({ onClose }: { onClose: () => void }) => { ...@@ -95,18 +99,9 @@ const PayModal = ({ onClose }: { onClose: () => void }) => {
}} }}
></Input> ></Input>
</Box> </Box>
<Markdown <Box flex={[1, '1 0 0']} overflow={'overlay'} px={6}>
source={` <Markdown source={priceMd} />
| 计费项 | 价格: 元/ 1K tokens(包含上下文)| </Box>
| --- | --- |
${vectorModelList
.map((item) => `| 索引-${item.name} | ${formatPrice(item.price, 1000)} |`)
.join('\n')}
${chatModelList
.map((item) => `| 对话-${item.name} | ${formatPrice(item.price, 1000)} |`)
.join('\n')}
| 文件QA拆分 | ${formatPrice(qaModel.price, 1000)} |`}
/>
</> </>
)} )}
{/* 付费二维码 */} {/* 付费二维码 */}
......
projects/app/src/pages/api/admin/initv43.ts
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction // Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import { PgClient } from '@/service/pg'; import { PgClient } from '@/service/pg';
import { PgDatasetTableName } from '@/constants/plugin'; import { PgDatasetTableName } from '@/constants/plugin';
......
projects/app/src/pages/api/admin/initv44.ts
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction // Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import { connectToDatabase, KB } from '@/service/mongo'; import { connectToDatabase } from '@/service/mongo';
import { KbTypeEnum } from '@/constants/dataset'; import { MongoDataset } from '@fastgpt/core/dataset/schema';
import { DatasetTypeEnum } from '@fastgpt/core/dataset/constant';
import { PgClient } from '@/service/pg'; import { PgClient } from '@/service/pg';
import { PgDatasetTableName } from '@/constants/plugin'; import { PgDatasetTableName } from '@/constants/plugin';
...@@ -12,13 +13,13 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) ...@@ -12,13 +13,13 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
await connectToDatabase(); await connectToDatabase();
await authUser({ req, authRoot: true }); await authUser({ req, authRoot: true });
await KB.updateMany( await MongoDataset.updateMany(
{ {
type: { $exists: false } type: { $exists: false }
}, },
{ {
$set: { $set: {
type: KbTypeEnum.dataset, type: DatasetTypeEnum.dataset,
parentId: null parentId: null
} }
} }
......
projects/app/src/pages/api/admin/initv441.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import { connectToDatabase } from '@/service/mongo'; import { connectToDatabase } from '@/service/mongo';
import mongoose from 'mongoose'; import mongoose from '@fastgpt/common/mongo';
import { PgClient } from '@/service/pg'; import { PgClient } from '@/service/pg';
import { PgDatasetTableName } from '@/constants/plugin'; import { PgDatasetTableName } from '@/constants/plugin';
......
projects/app/src/pages/api/admin/initv442.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import { connectToDatabase, Bill } from '@/service/mongo'; import { connectToDatabase, Bill } from '@/service/mongo';
export default async function handler(req: NextApiRequest, res: NextApiResponse) { export default async function handler(req: NextApiRequest, res: NextApiResponse) {
......
projects/app/src/pages/api/admin/initv445.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import { connectToDatabase, App } from '@/service/mongo'; import { connectToDatabase, App } from '@/service/mongo';
import { FlowInputItemTypeEnum, FlowModuleTypeEnum } from '@/constants/flow'; import { FlowInputItemTypeEnum, FlowModuleTypeEnum } from '@/constants/flow';
import { SystemInputEnum } from '@/constants/app'; import { SystemInputEnum } from '@/constants/app';
......
projects/app/src/pages/api/admin/initv447.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import { connectToDatabase } from '@/service/mongo'; import { connectToDatabase } from '@/service/mongo';
import { PgClient } from '@/service/pg'; import { PgClient } from '@/service/pg';
import { PgDatasetTableName } from '@/constants/plugin'; import { PgDatasetTableName } from '@/constants/plugin';
import { DatasetSpecialIdEnum } from '@fastgpt/core/dataset/constant'; import { DatasetSpecialIdEnum } from '@fastgpt/core/dataset/constant';
import mongoose, { Types } from 'mongoose'; import { Types, connectionMongo } from '@fastgpt/common/mongo';
import { delay } from '@/utils/tools'; import { delay } from '@/utils/tools';
export default async function handler(req: NextApiRequest, res: NextApiResponse) { export default async function handler(req: NextApiRequest, res: NextApiResponse) {
...@@ -71,7 +71,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) ...@@ -71,7 +71,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
} }
async function init(rows: any[], initFileIds: string[]) { async function init(rows: any[], initFileIds: string[]) {
const collection = mongoose.connection.db.collection(`dataset.files`); const collection = connectionMongo.connection.db.collection(`dataset.files`);
/* 遍历所有的 fileId，去找有没有对应的文件，没有的话则改成manual */ /* 遍历所有的 fileId，去找有没有对应的文件，没有的话则改成manual */
const updateResult = await Promise.allSettled( const updateResult = await Promise.allSettled(
......
projects/app/src/pages/api/admin/sqlTest.ts 0 → 100644
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { authUser } from '@fastgpt/support/user/auth';
import { connectToDatabase } from '@/service/mongo';
import { PgClient } from '@/service/pg';
import { PgDatasetTableName } from '@/constants/plugin';
import { DatasetSpecialIdEnum } from '@fastgpt/core/dataset/constant';
import { Types, connectionMongo } from '@fastgpt/common/mongo';
import { delay } from '@/utils/tools';
import { replaceVariable } from '@/utils/common/tools/text';
import { getVector } from '../openapi/plugin/vector';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
// await connectToDatabase();
// const { text, analyze, sql } = req.body as {
// userId: string;
// text: string;
// analyze?: boolean;
// sql: string;
// };
// await authUser({ req, authRoot: true });
// const vectorModel = global.vectorModels[0];
// const { vectors } = await getVector({
// model: vectorModel.model,
// input: [text]
// });
// const start = Date.now();
// const result: any = await PgClient.query(sql.replace(/\[vector\]/g, `[${vectors[0]}]`));
jsonRes(res, {
data: {
// rows: result?.[2]?.rows,
// time: Date.now() - start
}
});
} catch (error) {
jsonRes(res, {
code: 500,
error
});
}
}
projects/app/src/pages/api/app/create.ts
...@@ -2,13 +2,14 @@ ...@@ -2,13 +2,14 @@
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo'; import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import { App } from '@/service/models/app'; import { App } from '@/service/models/app';
import type { CreateAppParams } from '@/types/app'; import type { CreateAppParams } from '@/types/app';
import { AppTypeEnum } from '@/constants/app'; import { AppTypeEnum } from '@/constants/app';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try { try {
await connectToDatabase();
const { const {
name = 'APP', name = 'APP',
avatar, avatar,
...@@ -23,8 +24,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -23,8 +24,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
// 凭证校验 // 凭证校验
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
// 上限校验 // 上限校验
const authCount = await App.countDocuments({ const authCount = await App.countDocuments({
userId userId
......
projects/app/src/pages/api/app/data/totalUsage.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase, Bill } from '@/service/mongo'; import { connectToDatabase, Bill } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import { Types } from 'mongoose'; import { Types } from '@fastgpt/common/mongo';
export default async function handler(req: NextApiRequest, res: NextApiResponse) { export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try { try {
await connectToDatabase();
const { appId, start, end } = req.body as { appId: string; start: number; end: number }; const { appId, start, end } = req.body as { appId: string; start: number; end: number };
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
const result = await Bill.aggregate([ const result = await Bill.aggregate([
{ {
$match: { $match: {
......
projects/app/src/pages/api/app/del.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { Chat, App, connectToDatabase, Collection, OutLink } from '@/service/mongo'; import { Chat, App, connectToDatabase, Collection } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { MongoOutLink } from '@fastgpt/support/outLink/schema';
import { authUser } from '@fastgpt/support/user/auth';
import { authApp } from '@/service/utils/auth'; import { authApp } from '@/service/utils/auth';
/* 获取我的模型 */ /* 获取我的模型 */
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try { try {
await connectToDatabase();
const { appId } = req.query as { appId: string }; const { appId } = req.query as { appId: string };
if (!appId) { if (!appId) {
...@@ -16,8 +18,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -16,8 +18,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
// 凭证校验 // 凭证校验
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
// 验证是否是该用户的 app // 验证是否是该用户的 app
await authApp({ await authApp({
appId, appId,
...@@ -35,7 +35,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -35,7 +35,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
}); });
// 删除分享链接 // 删除分享链接
await OutLink.deleteMany({ await MongoOutLink.deleteMany({
appId appId
}); });
......
projects/app/src/pages/api/app/detail.tsx
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo'; import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import { authApp } from '@/service/utils/auth'; import { authApp } from '@/service/utils/auth';
/* 获取我的模型 */ /* 获取我的模型 */
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try { try {
await connectToDatabase();
const { appId } = req.query as { appId: string }; const { appId } = req.query as { appId: string };
if (!appId) { if (!appId) {
...@@ -16,8 +17,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -16,8 +17,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
// 凭证校验 // 凭证校验
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
const { app } = await authApp({ const { app } = await authApp({
appId, appId,
userId userId
......
projects/app/src/pages/api/app/getChatLogs.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { Chat, connectToDatabase } from '@/service/mongo'; import { Chat, connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import type { PagingData } from '@/types'; import type { PagingData } from '@/types';
import { AppLogsListItemType } from '@/types/app'; import { AppLogsListItemType } from '@/types/app';
import { Types } from 'mongoose'; import { Types } from '@fastgpt/common/mongo';
import { addDays } from 'date-fns'; import { addDays } from 'date-fns';
import type { GetAppChatLogsParams } from '@/global/core/api/appReq.d'; import type { GetAppChatLogsParams } from '@/global/core/api/appReq.d';
export default async function handler(req: NextApiRequest, res: NextApiResponse) { export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try { try {
await connectToDatabase();
const { const {
pageNum = 1, pageNum = 1,
pageSize = 20, pageSize = 20,
...@@ -21,7 +22,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) ...@@ -21,7 +22,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
if (!appId) { if (!appId) {
throw new Error('缺少参数'); throw new Error('缺少参数');
} }
await connectToDatabase();
// 凭证校验 // 凭证校验
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
......
projects/app/src/pages/api/app/myApps.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase, App } from '@/service/mongo'; import { connectToDatabase, App } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import { AppListItemType } from '@/types/app'; import { AppListItemType } from '@/types/app';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try { try {
await connectToDatabase();
// 凭证校验 // 凭证校验
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
// 根据 userId 获取模型信息 // 根据 userId 获取模型信息
const myApps = await App.find( const myApps = await App.find(
{ {
......
projects/app/src/pages/api/app/share/collection.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase, Collection, App } from '@/service/mongo'; import { connectToDatabase, Collection, App } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
/* 模型收藏切换 */ /* 模型收藏切换 */
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try { try {
await connectToDatabase();
const { appId } = req.query as { appId: string }; const { appId } = req.query as { appId: string };
if (!appId) { if (!appId) {
...@@ -14,8 +15,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -14,8 +15,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
// 凭证校验 // 凭证校验
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
const collectionRecord = await Collection.findOne({ const collectionRecord = await Collection.findOne({
userId, userId,
modelId: appId modelId: appId
......
projects/app/src/pages/api/app/share/getModels.ts
...@@ -3,20 +3,19 @@ import { jsonRes } from '@/service/response'; ...@@ -3,20 +3,19 @@ import { jsonRes } from '@/service/response';
import { connectToDatabase, App } from '@/service/mongo'; import { connectToDatabase, App } from '@/service/mongo';
import type { PagingData } from '@/types'; import type { PagingData } from '@/types';
import type { ShareAppItem } from '@/types/app'; import type { ShareAppItem } from '@/types/app';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import { Types } from 'mongoose'; import { Types } from '@fastgpt/common/mongo';
/* 获取模型列表 */ /* 获取模型列表 */
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try { try {
await connectToDatabase();
const { const {
searchText = '', searchText = '',
pageNum = 1, pageNum = 1,
pageSize = 20 pageSize = 20
} = req.body as { searchText: string; pageNum: number; pageSize: number }; } = req.body as { searchText: string; pageNum: number; pageSize: number };
await connectToDatabase();
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
const regex = new RegExp(searchText, 'i'); const regex = new RegExp(searchText, 'i');
......
projects/app/src/pages/api/app/update.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo'; import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import { App } from '@/service/models/app'; import { App } from '@/service/models/app';
import type { AppUpdateParams } from '@/types/app'; import type { AppUpdateParams } from '@/types/app';
import { authApp } from '@/service/utils/auth'; import { authApp } from '@/service/utils/auth';
...@@ -9,6 +9,7 @@ import { authApp } from '@/service/utils/auth'; ...@@ -9,6 +9,7 @@ import { authApp } from '@/service/utils/auth';
/* 获取我的模型 */ /* 获取我的模型 */
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try { try {
await connectToDatabase();
const { name, avatar, type, share, intro, modules } = req.body as AppUpdateParams; const { name, avatar, type, share, intro, modules } = req.body as AppUpdateParams;
const { appId } = req.query as { appId: string }; const { appId } = req.query as { appId: string };
...@@ -19,8 +20,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -19,8 +20,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
// 凭证校验 // 凭证校验
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
await authApp({ await authApp({
appId, appId,
userId userId
......
projects/app/src/pages/api/chat/chatTest.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { connectToDatabase } from '@/service/mongo'; import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import { sseErrRes } from '@/service/response'; import { sseErrRes } from '@/service/response';
import { sseResponseEventEnum } from '@/constants/chat'; import { sseResponseEventEnum } from '@/constants/chat';
import { sseResponse } from '@/service/utils/tools'; import { sseResponse } from '@/service/utils/tools';
...@@ -30,6 +30,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) ...@@ -30,6 +30,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
let { modules = [], history = [], prompt, variables = {}, appName, appId } = req.body as Props; let { modules = [], history = [], prompt, variables = {}, appName, appId } = req.body as Props;
try { try {
await connectToDatabase();
if (!history || !modules || !prompt) { if (!history || !modules || !prompt) {
throw new Error('Prams Error'); throw new Error('Prams Error');
} }
...@@ -37,8 +38,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) ...@@ -37,8 +38,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
throw new Error('history is not array'); throw new Error('history is not array');
} }
await connectToDatabase();
/* user auth */ /* user auth */
const { userId, user } = await authUser({ req, authToken: true, authBalance: true }); const { userId, user } = await authUser({ req, authToken: true, authBalance: true });
......
projects/app/src/pages/api/chat/delChatRecordByContentId.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase, ChatItem } from '@/service/mongo'; import { connectToDatabase, ChatItem } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
export default async function handler(req: NextApiRequest, res: NextApiResponse) { export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try { try {
const { chatId, contentId } = req.query as { chatId: string; contentId: string };
await connectToDatabase(); await connectToDatabase();
const { chatId, contentId } = req.query as { chatId: string; contentId: string };
// 凭证校验 // 凭证校验
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
......
projects/app/src/pages/api/chat/feedback/adminUpdate.ts
...@@ -2,7 +2,7 @@ import type { NextApiRequest, NextApiResponse } from 'next'; ...@@ -2,7 +2,7 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase, ChatItem } from '@/service/mongo'; import { connectToDatabase, ChatItem } from '@/service/mongo';
import type { AdminUpdateFeedbackParams } from '@/global/core/api/chatReq.d'; import type { AdminUpdateFeedbackParams } from '@/global/core/api/chatReq.d';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
/* 初始化我的聊天框，需要身份验证 */ /* 初始化我的聊天框，需要身份验证 */
export default async function handler(req: NextApiRequest, res: NextApiResponse) { export default async function handler(req: NextApiRequest, res: NextApiResponse) {
......
projects/app/src/pages/api/chat/history/getHistory.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase, Chat } from '@/service/mongo'; import { connectToDatabase, Chat } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import type { ChatHistoryItemType } from '@/types/chat'; import type { ChatHistoryItemType } from '@/types/chat';
import { ChatSourceEnum } from '@/constants/chat'; import { ChatSourceEnum } from '@/constants/chat';
/* 获取历史记录 */ /* 获取历史记录 */
export default async function handler(req: NextApiRequest, res: NextApiResponse) { export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try { try {
await connectToDatabase();
const { appId } = req.body as { appId?: string }; const { appId } = req.body as { appId?: string };
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
const data = await Chat.find( const data = await Chat.find(
{ {
userId, userId,
......
projects/app/src/pages/api/chat/history/updateChatHistory.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase, Chat } from '@/service/mongo'; import { connectToDatabase, Chat } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
export type Props = { export type Props = {
chatId: string; chatId: string;
...@@ -12,12 +12,11 @@ export type Props = { ...@@ -12,12 +12,11 @@ export type Props = {
/* 更新聊天标题 */ /* 更新聊天标题 */
export default async function handler(req: NextApiRequest, res: NextApiResponse) { export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try { try {
await connectToDatabase();
const { chatId, customTitle, top } = req.body as Props; const { chatId, customTitle, top } = req.body as Props;
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
await Chat.findOneAndUpdate( await Chat.findOneAndUpdate(
{ {
chatId, chatId,
......
projects/app/src/pages/api/chat/init.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { Chat, ChatItem } from '@/service/mongo'; import { Chat, ChatItem, connectToDatabase } from '@/service/mongo';
import type { InitChatResponse } from '@/global/core/api/chatRes.d'; import type { InitChatResponse } from '@/global/core/api/chatRes.d';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import { ChatItemType } from '@/types/chat'; import { ChatItemType } from '@/types/chat';
import { authApp } from '@/service/utils/auth'; import { authApp } from '@/service/utils/auth';
import type { ChatSchema } from '@/types/mongoSchema'; import type { ChatSchema } from '@/types/mongoSchema';
...@@ -12,6 +12,7 @@ import { TaskResponseKeyEnum } from '@/constants/chat'; ...@@ -12,6 +12,7 @@ import { TaskResponseKeyEnum } from '@/constants/chat';
/* 初始化我的聊天框，需要身份验证 */ /* 初始化我的聊天框，需要身份验证 */
export default async function handler(req: NextApiRequest, res: NextApiResponse) { export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try { try {
await connectToDatabase();
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
let { appId, chatId } = req.query as { let { appId, chatId } = req.query as {
......
projects/app/src/pages/api/chat/removeHistory.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase, Chat, ChatItem } from '@/service/mongo'; import { connectToDatabase, Chat, ChatItem } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import { ChatSourceEnum } from '@/constants/chat'; import { ChatSourceEnum } from '@/constants/chat';
type Props = { type Props = {
...@@ -12,11 +12,10 @@ type Props = { ...@@ -12,11 +12,10 @@ type Props = {
/* clear chat history */ /* clear chat history */
export default async function handler(req: NextApiRequest, res: NextApiResponse) { export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try { try {
await connectToDatabase();
const { chatId, appId } = req.query as Props; const { chatId, appId } = req.query as Props;
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
if (chatId) { if (chatId) {
await Promise.all([ await Promise.all([
Chat.findOneAndRemove({ Chat.findOneAndRemove({
......
projects/app/src/pages/api/common/bill/createTrainingBill.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase, Bill } from '@/service/mongo'; import { connectToDatabase, Bill } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import { BillSourceEnum } from '@/constants/user'; import { BillSourceEnum } from '@/constants/user';
import { CreateTrainingBillType } from '@/global/common/api/billReq.d'; import { CreateTrainingBillType } from '@/global/common/api/billReq.d';
export default async function handler(req: NextApiRequest, res: NextApiResponse) { export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try { try {
await connectToDatabase();
const { name } = req.body as CreateTrainingBillType; const { name } = req.body as CreateTrainingBillType;
const { userId } = await authUser({ req, authToken: true, authApiKey: true }); const { userId } = await authUser({ req, authToken: true, authApiKey: true });
await connectToDatabase();
const { _id } = await Bill.create({ const { _id } = await Bill.create({
userId, userId,
appName: name, appName: name,
......
projects/app/src/pages/api/core/ai/agent/createQuestionGuide.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo'; import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import type { CreateQuestionGuideParams } from '@/global/core/api/aiReq.d'; import type { CreateQuestionGuideParams } from '@/global/core/api/aiReq.d';
import { pushQuestionGuideBill } from '@/service/common/bill/push'; import { pushQuestionGuideBill } from '@/service/common/bill/push';
import { defaultQGModel } from '@/pages/api/system/getInitData'; import { defaultQGModel } from '@/pages/api/system/getInitData';
......
projects/app/src/pages/api/core/dataset/allDataset.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase, KB } from '@/service/mongo'; import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { MongoDataset } from '@fastgpt/core/dataset/schema';
import { authUser } from '@fastgpt/support/user/auth';
import { getVectorModel } from '@/service/utils/data'; import { getVectorModel } from '@/service/utils/data';
import type { DatasetsItemType } from '@/types/core/dataset'; import type { DatasetsItemType } from '@/types/core/dataset';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try { try {
await connectToDatabase();
// 凭证校验 // 凭证校验
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
await connectToDatabase(); const kbList = await MongoDataset.find({
const kbList = await KB.find({
userId, userId,
type: 'dataset' type: 'dataset'
}); });
......
projects/app/src/pages/api/core/dataset/create.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase, KB } from '@/service/mongo'; import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { MongoDataset } from '@fastgpt/core/dataset/schema';
import { authUser } from '@fastgpt/support/user/auth';
import type { CreateDatasetParams } from '@/global/core/api/datasetReq.d'; import type { CreateDatasetParams } from '@/global/core/api/datasetReq.d';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try { try {
await connectToDatabase();
const { name, tags, avatar, vectorModel, parentId, type } = req.body as CreateDatasetParams; const { name, tags, avatar, vectorModel, parentId, type } = req.body as CreateDatasetParams;
// 凭证校验 // 凭证校验
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
await connectToDatabase(); const { _id } = await MongoDataset.create({
const { _id } = await KB.create({
name, name,
userId, userId,
tags, tags,
......
projects/app/src/pages/api/core/dataset/data/delDataById.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import { PgClient } from '@/service/pg'; import { PgClient } from '@/service/pg';
import { withNextCors } from '@/service/utils/tools'; import { withNextCors } from '@/service/utils/tools';
import { PgDatasetTableName } from '@/constants/plugin'; import { PgDatasetTableName } from '@/constants/plugin';
import { connectToDatabase } from '@/service/mongo';
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try { try {
await connectToDatabase();
let { dataId } = req.query as { let { dataId } = req.query as {
dataId: string; dataId: string;
}; };
......
projects/app/src/pages/api/core/dataset/data/exportAll.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase, User } from '@/service/mongo'; import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { MongoUser } from '@fastgpt/support/user/schema';
import { authUser } from '@fastgpt/support/user/auth';
import { PgDatasetTableName } from '@/constants/plugin'; import { PgDatasetTableName } from '@/constants/plugin';
import { findAllChildrenIds } from '../delete'; import { findAllChildrenIds } from '../delete';
import QueryStream from 'pg-query-stream'; import QueryStream from 'pg-query-stream';
...@@ -11,6 +12,7 @@ import { responseWriteController } from '@/service/common/stream'; ...@@ -11,6 +12,7 @@ import { responseWriteController } from '@/service/common/stream';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try { try {
await connectToDatabase();
let { kbId } = req.query as { let { kbId } = req.query as {
kbId: string; kbId: string;
}; };
...@@ -19,8 +21,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -19,8 +21,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
throw new Error('缺少参数'); throw new Error('缺少参数');
} }
await connectToDatabase();
// 凭证校验 // 凭证校验
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
...@@ -31,7 +31,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -31,7 +31,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
); );
// auth export times // auth export times
const authTimes = await User.findOne( const authTimes = await MongoUser.findOne(
{ {
_id: userId, _id: userId,
$or: [ $or: [
...@@ -101,7 +101,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -101,7 +101,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
stream.on('end', async () => { stream.on('end', async () => {
try { try {
// update export time // update export time
await User.findByIdAndUpdate(userId, { await MongoUser.findByIdAndUpdate(userId, {
'limit.exportKbTime': new Date() 'limit.exportKbTime': new Date()
}); });
} catch (error) {} } catch (error) {}
......
projects/app/src/pages/api/core/dataset/data/getDataById.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo'; import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import { PgClient } from '@/service/pg'; import { PgClient } from '@/service/pg';
import { PgDatasetTableName } from '@/constants/plugin'; import { PgDatasetTableName } from '@/constants/plugin';
import type { PgDataItemType } from '@/types/core/dataset/data'; import type { PgDataItemType } from '@/types/core/dataset/data';
...@@ -15,6 +15,7 @@ export type Response = { ...@@ -15,6 +15,7 @@ export type Response = {
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try { try {
await connectToDatabase();
let { dataId } = req.query as { let { dataId } = req.query as {
dataId: string; dataId: string;
}; };
...@@ -25,8 +26,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -25,8 +26,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
// 凭证校验 // 凭证校验
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
const where: any = [['user_id', userId], 'AND', ['id', dataId]]; const where: any = [['user_id', userId], 'AND', ['id', dataId]];
const searchRes = await PgClient.select<PgDataItemType>(PgDatasetTableName, { const searchRes = await PgClient.select<PgDataItemType>(PgDatasetTableName, {
......
projects/app/src/pages/api/core/dataset/data/getDataList.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo'; import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import { PgClient } from '@/service/pg'; import { PgClient } from '@/service/pg';
import { PgDatasetTableName } from '@/constants/plugin'; import { PgDatasetTableName } from '@/constants/plugin';
import type { PgDataItemType } from '@/types/core/dataset/data'; import type { PgDataItemType } from '@/types/core/dataset/data';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try { try {
await connectToDatabase();
let { let {
kbId, kbId,
pageNum = 1, pageNum = 1,
...@@ -28,7 +29,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -28,7 +29,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
// 凭证校验 // 凭证校验
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
searchText = searchText.replace(/'/g, ''); searchText = searchText.replace(/'/g, '');
const where: any = [ const where: any = [
......
projects/app/src/pages/api/core/dataset/data/getQueueLen.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { TrainingData } from '@/service/mongo'; import { TrainingData, connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
/* 拆分数据成QA */ /* 拆分数据成QA */
export default async function handler(req: NextApiRequest, res: NextApiResponse) { export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try { try {
await connectToDatabase();
await authUser({ req, authToken: true }); await authUser({ req, authToken: true });
// split queue data // split queue data
......
projects/app/src/pages/api/core/dataset/data/getTrainingData.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase, TrainingData } from '@/service/mongo'; import { connectToDatabase, TrainingData } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import { TrainingModeEnum } from '@/constants/plugin'; import { TrainingModeEnum } from '@/constants/plugin';
import { Types } from 'mongoose'; import { Types } from '@fastgpt/common/mongo';
import { startQueue } from '@/service/utils/tools'; import { startQueue } from '@/service/utils/tools';
/* 拆分数据成QA */ /* 拆分数据成QA */
export default async function handler(req: NextApiRequest, res: NextApiResponse) { export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try { try {
await connectToDatabase();
const { kbId, init = false } = req.body as { kbId: string; init: boolean }; const { kbId, init = false } = req.body as { kbId: string; init: boolean };
if (!kbId) { if (!kbId) {
throw new Error('参数错误'); throw new Error('参数错误');
} }
await connectToDatabase();
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
......
projects/app/src/pages/api/core/dataset/data/insertData.ts
...@@ -5,7 +5,8 @@ ...@@ -5,7 +5,8 @@
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo'; import { connectToDatabase } from '@/service/mongo';
import { authKb, authUser } from '@/service/utils/auth'; import { authDataset } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
import { withNextCors } from '@/service/utils/tools'; import { withNextCors } from '@/service/utils/tools';
import { PgDatasetTableName } from '@/constants/plugin'; import { PgDatasetTableName } from '@/constants/plugin';
import { insertData2Dataset, PgClient } from '@/service/pg'; import { insertData2Dataset, PgClient } from '@/service/pg';
...@@ -50,7 +51,7 @@ export async function getVectorAndInsertDataset( ...@@ -50,7 +51,7 @@ export async function getVectorAndInsertDataset(
} }
// auth kb // auth kb
const kb = await authKb({ kbId, userId }); const kb = await authDataset({ kbId, userId });
const q = data?.q?.replace(/\\n/g, '\n').trim().replace(/'/g, '"'); const q = data?.q?.replace(/\\n/g, '\n').trim().replace(/'/g, '"');
const a = data?.a?.replace(/\\n/g, '\n').trim().replace(/'/g, '"'); const a = data?.a?.replace(/\\n/g, '\n').trim().replace(/'/g, '"');
......
projects/app/src/pages/api/core/dataset/data/pushData.ts
/* push data to training queue */ /* push data to training queue */
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase, TrainingData, KB } from '@/service/mongo'; import { connectToDatabase, TrainingData } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { MongoDataset } from '@fastgpt/core/dataset/schema';
import { authKb } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import { authDataset } from '@/service/utils/auth';
import { withNextCors } from '@/service/utils/tools'; import { withNextCors } from '@/service/utils/tools';
import { TrainingModeEnum } from '@/constants/plugin'; import { TrainingModeEnum } from '@/constants/plugin';
import { startQueue } from '@/service/utils/tools'; import { startQueue } from '@/service/utils/tools';
...@@ -21,6 +22,7 @@ const modeMap = { ...@@ -21,6 +22,7 @@ const modeMap = {
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try { try {
await connectToDatabase();
const { kbId, data, mode = TrainingModeEnum.index } = req.body as PushDataProps; const { kbId, data, mode = TrainingModeEnum.index } = req.body as PushDataProps;
if (!kbId || !Array.isArray(data)) { if (!kbId || !Array.isArray(data)) {
...@@ -35,8 +37,6 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex ...@@ -35,8 +37,6 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
throw new Error('Data is too long, max 500'); throw new Error('Data is too long, max 500');
} }
await connectToDatabase();
// 凭证校验 // 凭证校验
const { userId } = await authUser({ req, authToken: true, authApiKey: true }); const { userId } = await authUser({ req, authToken: true, authApiKey: true });
...@@ -63,13 +63,13 @@ export async function pushDataToKb({ ...@@ -63,13 +63,13 @@ export async function pushDataToKb({
billId billId
}: { userId: string } & PushDataProps): Promise<PushDataResponse> { }: { userId: string } & PushDataProps): Promise<PushDataResponse> {
const [kb, vectorModel] = await Promise.all([ const [kb, vectorModel] = await Promise.all([
authKb({ authDataset({
userId, userId,
kbId kbId
}), }),
(async () => { (async () => {
if (mode === TrainingModeEnum.index) { if (mode === TrainingModeEnum.index) {
const vectorModel = (await KB.findById(kbId, 'vectorModel'))?.vectorModel; const vectorModel = (await MongoDataset.findById(kbId, 'vectorModel'))?.vectorModel;
return getVectorModel(vectorModel || global.vectorModels[0].model); return getVectorModel(vectorModel || global.vectorModels[0].model);
} }
......
projects/app/src/pages/api/core/dataset/data/updateData.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import { PgClient } from '@/service/pg'; import { PgClient } from '@/service/pg';
import { withNextCors } from '@/service/utils/tools'; import { withNextCors } from '@/service/utils/tools';
import { KB, connectToDatabase } from '@/service/mongo'; import { connectToDatabase } from '@/service/mongo';
import { MongoDataset } from '@fastgpt/core/dataset/schema';
import { getVector } from '@/pages/api/openapi/plugin/vector'; import { getVector } from '@/pages/api/openapi/plugin/vector';
import { PgDatasetTableName } from '@/constants/plugin'; import { PgDatasetTableName } from '@/constants/plugin';
import type { UpdateDatasetDataPrams } from '@/global/core/api/datasetReq.d'; import type { UpdateDatasetDataPrams } from '@/global/core/api/datasetReq.d';
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try { try {
await connectToDatabase();
const { dataId, a = '', q = '', kbId } = req.body as UpdateDatasetDataPrams; const { dataId, a = '', q = '', kbId } = req.body as UpdateDatasetDataPrams;
if (!dataId) { if (!dataId) {
throw new Error('缺少参数'); throw new Error('缺少参数');
} }
await connectToDatabase();
// auth user and get kb // auth user and get kb
const [{ userId }, kb] = await Promise.all([ const [{ userId }, kb] = await Promise.all([
authUser({ req, authToken: true }), authUser({ req, authToken: true }),
KB.findById(kbId, 'vectorModel') MongoDataset.findById(kbId, 'vectorModel')
]); ]);
if (!kb) { if (!kb) {
......
projects/app/src/pages/api/core/dataset/delete.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase, KB, TrainingData } from '@/service/mongo'; import { connectToDatabase, TrainingData } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { MongoDataset } from '@fastgpt/core/dataset/schema';
import { authUser } from '@fastgpt/support/user/auth';
import { PgClient } from '@/service/pg'; import { PgClient } from '@/service/pg';
import { PgDatasetTableName } from '@/constants/plugin'; import { PgDatasetTableName } from '@/constants/plugin';
import { GridFSStorage } from '@/service/lib/gridfs'; import { GridFSStorage } from '@/service/lib/gridfs';
import { Types } from 'mongoose'; import { Types } from '@fastgpt/common/mongo';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try { try {
...@@ -43,7 +44,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -43,7 +44,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
await Promise.all(deletedIds.map((id) => gridFs.deleteFilesByKbId(id))); await Promise.all(deletedIds.map((id) => gridFs.deleteFilesByKbId(id)));
// delete kb data // delete kb data
await KB.deleteMany({ await MongoDataset.deleteMany({
_id: { $in: deletedIds }, _id: { $in: deletedIds },
userId userId
}); });
...@@ -59,7 +60,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -59,7 +60,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
export async function findAllChildrenIds(id: string) { export async function findAllChildrenIds(id: string) {
// find children // find children
const children = await KB.find({ parentId: id }); const children = await MongoDataset.find({ parentId: id });
let allChildrenIds = children.map((child) => String(child._id)); let allChildrenIds = children.map((child) => String(child._id));
......
projects/app/src/pages/api/core/dataset/detail.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase, KB } from '@/service/mongo'; import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import { getVectorModel } from '@/service/utils/data'; import { getVectorModel } from '@/service/utils/data';
import { MongoDataset } from '@fastgpt/core/dataset/schema';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try { try {
await connectToDatabase();
const { id } = req.query as { const { id } = req.query as {
id: string; id: string;
}; };
...@@ -17,9 +19,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -17,9 +19,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
// 凭证校验 // 凭证校验
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
await connectToDatabase(); const data = await MongoDataset.findOne({
const data = await KB.findOne({
_id: id, _id: id,
userId userId
}); });
......
projects/app/src/pages/api/core/dataset/file/delById.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase, TrainingData } from '@/service/mongo'; import { connectToDatabase, TrainingData } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import { GridFSStorage } from '@/service/lib/gridfs'; import { GridFSStorage } from '@/service/lib/gridfs';
import { PgClient } from '@/service/pg'; import { PgClient } from '@/service/pg';
import { PgDatasetTableName } from '@/constants/plugin'; import { PgDatasetTableName } from '@/constants/plugin';
import { Types } from 'mongoose'; import { Types } from '@fastgpt/common/mongo';
import { isSpecialFileId } from '@fastgpt/core/dataset/utils'; import { isSpecialFileId } from '@fastgpt/core/dataset/utils';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
......
projects/app/src/pages/api/core/dataset/file/delEmptyFiles.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo'; import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import { GridFSStorage } from '@/service/lib/gridfs'; import { GridFSStorage } from '@/service/lib/gridfs';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
......
projects/app/src/pages/api/core/dataset/file/detail.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo'; import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import { GridFSStorage } from '@/service/lib/gridfs'; import { GridFSStorage } from '@/service/lib/gridfs';
import { datasetSpecialIdMap } from '@fastgpt/core/dataset/constant'; import { datasetSpecialIdMap } from '@fastgpt/core/dataset/constant';
import { datasetSpecialIds } from '@fastgpt/core/dataset/constant'; import { datasetSpecialIds } from '@fastgpt/core/dataset/constant';
......
projects/app/src/pages/api/core/dataset/file/list.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase, TrainingData } from '@/service/mongo'; import { connectToDatabase, TrainingData } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import { GridFSStorage } from '@/service/lib/gridfs'; import { GridFSStorage } from '@/service/lib/gridfs';
import { PgClient, updateDataFileId } from '@/service/pg'; import { PgClient, updateDataFileId } from '@/service/pg';
import { PgDatasetTableName } from '@/constants/plugin'; import { PgDatasetTableName } from '@/constants/plugin';
import { FileStatusEnum } from '@/constants/dataset';
import { strIsLink } from '@fastgpt/common/tools/str'; import { strIsLink } from '@fastgpt/common/tools/str';
import { import {
FileStatusEnum,
DatasetSpecialIdEnum, DatasetSpecialIdEnum,
datasetSpecialIdMap, datasetSpecialIdMap,
datasetSpecialIds datasetSpecialIds
} from '@fastgpt/core/dataset/constant'; } from '@fastgpt/core/dataset/constant';
import { Types } from 'mongoose'; import { Types } from '@fastgpt/common/mongo';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try { try {
......
projects/app/src/pages/api/core/dataset/file/markUsed.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo'; import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import { GridFSStorage } from '@/service/lib/gridfs'; import { GridFSStorage } from '@/service/lib/gridfs';
import { MarkFileUsedProps } from '@/global/core/api/datasetReq.d'; import { MarkFileUsedProps } from '@/global/core/api/datasetReq.d';
import { Types } from 'mongoose'; import { Types } from '@fastgpt/common/mongo';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try { try {
......
projects/app/src/pages/api/core/dataset/file/update.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo'; import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import { GridFSStorage } from '@/service/lib/gridfs'; import { GridFSStorage } from '@/service/lib/gridfs';
import { UpdateFileProps } from '@/global/core/api/datasetReq.d'; import { UpdateFileProps } from '@/global/core/api/datasetReq.d';
import { Types } from 'mongoose'; import { Types } from '@fastgpt/common/mongo';
import { PgClient } from '@/service/pg'; import { PgClient } from '@/service/pg';
import { PgDatasetTableName } from '@/constants/plugin'; import { PgDatasetTableName } from '@/constants/plugin';
import { addLog } from '@/service/utils/tools'; import { addLog } from '@/service/utils/tools';
......
projects/app/src/pages/api/core/dataset/list.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase, KB } from '@/service/mongo'; import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import { getVectorModel } from '@/service/utils/data'; import { getVectorModel } from '@/service/utils/data';
import type { DatasetsItemType } from '@/types/core/dataset'; import type { DatasetsItemType } from '@/types/core/dataset';
import { KbTypeEnum } from '@/constants/dataset'; import { DatasetTypeEnum } from '@fastgpt/core/dataset/constant';
import { MongoDataset } from '@fastgpt/core/dataset/schema';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try { try {
const { parentId, type } = req.query as { parentId?: string; type?: `${KbTypeEnum}` }; await connectToDatabase();
const { parentId, type } = req.query as { parentId?: string; type?: `${DatasetTypeEnum}` };
// 凭证校验 // 凭证校验
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
await connectToDatabase(); const kbList = await MongoDataset.find({
const kbList = await KB.find({
userId, userId,
...(parentId !== undefined && { parentId: parentId || null }), ...(parentId !== undefined && { parentId: parentId || null }),
...(type && { type }) ...(type && { type })
......
projects/app/src/pages/api/core/dataset/paths.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase, KB } from '@/service/mongo'; import { connectToDatabase } from '@/service/mongo';
import { MongoDataset } from '@fastgpt/core/dataset/schema';
import type { DatasetPathItemType } from '@/types/core/dataset'; import type { DatasetPathItemType } from '@/types/core/dataset';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
...@@ -25,7 +26,7 @@ async function getParents(parentId?: string): Promise<DatasetPathItemType[]> { ...@@ -25,7 +26,7 @@ async function getParents(parentId?: string): Promise<DatasetPathItemType[]> {
return []; return [];
} }
const parent = await KB.findById(parentId, 'name parentId'); const parent = await MongoDataset.findById(parentId, 'name parentId');
if (!parent) return []; if (!parent) return [];
......
projects/app/src/pages/api/core/dataset/searchTest.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import { PgClient } from '@/service/pg'; import { PgClient } from '@/service/pg';
import { withNextCors } from '@/service/utils/tools'; import { withNextCors } from '@/service/utils/tools';
import { getVector } from '../../openapi/plugin/vector'; import { getVector } from '../../openapi/plugin/vector';
import { PgDatasetTableName } from '@/constants/plugin'; import { PgDatasetTableName } from '@/constants/plugin';
import { KB } from '@/service/mongo'; import { MongoDataset } from '@fastgpt/core/dataset/schema';
import type { SearchTestProps } from '@/global/core/api/datasetReq.d'; import type { SearchTestProps } from '@/global/core/api/datasetReq.d';
import type { SearchTestResponseType } from '@/global/core/api/datasetRes.d'; import type { SearchTestResponseType } from '@/global/core/api/datasetRes.d';
import { connectToDatabase } from '@/service/mongo';
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try { try {
await connectToDatabase();
const { kbId, text } = req.body as SearchTestProps; const { kbId, text } = req.body as SearchTestProps;
if (!kbId || !text) { if (!kbId || !text) {
...@@ -20,7 +22,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex ...@@ -20,7 +22,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
// 凭证校验 // 凭证校验
const [{ userId }, kb] = await Promise.all([ const [{ userId }, kb] = await Promise.all([
authUser({ req, authToken: true, authApiKey: true }), authUser({ req, authToken: true, authApiKey: true }),
KB.findById(kbId, 'vectorModel') MongoDataset.findById(kbId, 'vectorModel')
]); ]);
if (!userId || !kb) { if (!userId || !kb) {
...@@ -35,7 +37,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex ...@@ -35,7 +37,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
const response: any = await PgClient.query( const response: any = await PgClient.query(
`BEGIN; `BEGIN;
SET LOCAL ivfflat.probes = ${global.systemEnv.pgIvfflatProbe || 10}; SET LOCAL hnsw.ef_search= ${global.systemEnv.pgHNSWEfSearch || 40};
select id, q, a, source, file_id, (vector <#> '[${ select id, q, a, source, file_id, (vector <#> '[${
vectors[0] vectors[0]
}]') * -1 AS score from ${PgDatasetTableName} where kb_id='${kbId}' AND user_id='${userId}' order by vector <#> '[${ }]') * -1 AS score from ${PgDatasetTableName} where kb_id='${kbId}' AND user_id='${userId}' order by vector <#> '[${
......
projects/app/src/pages/api/core/dataset/update.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase, KB } from '@/service/mongo'; import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { MongoDataset } from '@fastgpt/core/dataset/schema';
import { authUser } from '@fastgpt/support/user/auth';
import type { DatasetUpdateParams } from '@/global/core/api/datasetReq.d'; import type { DatasetUpdateParams } from '@/global/core/api/datasetReq.d';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try { try {
await connectToDatabase();
const { id, parentId, name, avatar, tags } = req.body as DatasetUpdateParams; const { id, parentId, name, avatar, tags } = req.body as DatasetUpdateParams;
if (!id) { if (!id) {
...@@ -15,9 +17,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -15,9 +17,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
// 凭证校验 // 凭证校验
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
await connectToDatabase(); await MongoDataset.findOneAndUpdate(
await KB.findOneAndUpdate(
{ {
_id: id, _id: id,
userId userId
......
projects/app/src/pages/api/openapi/plugin/vector.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { authBalanceByUid, authUser } from '@/service/utils/auth'; import { authBalanceByUid, authUser } from '@fastgpt/support/user/auth';
import { withNextCors } from '@/service/utils/tools'; import { withNextCors } from '@/service/utils/tools';
import { getAIApi } from '@fastgpt/core/ai/config'; import { getAIApi } from '@fastgpt/core/ai/config';
import { pushGenerateVectorBill } from '@/service/common/bill/push'; import { pushGenerateVectorBill } from '@/service/common/bill/push';
import { connectToDatabase } from '@/service/mongo';
type Props = { type Props = {
model: string; model: string;
...@@ -17,6 +18,7 @@ type Response = { ...@@ -17,6 +18,7 @@ type Response = {
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try { try {
await connectToDatabase();
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
let { input, model } = req.query as Props; let { input, model } = req.query as Props;
......
projects/app/src/pages/api/plugins/urlFetch.ts
...@@ -4,14 +4,16 @@ import axios from 'axios'; ...@@ -4,14 +4,16 @@ import axios from 'axios';
import { JSDOM } from 'jsdom'; import { JSDOM } from 'jsdom';
import { Readability } from '@mozilla/readability'; import { Readability } from '@mozilla/readability';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import type { FetchResultItem } from '@/global/common/api/pluginRes.d'; import type { FetchResultItem } from '@/global/common/api/pluginRes.d';
import { simpleText } from '@/utils/file'; import { simpleText } from '@fastgpt/common/tools/str';
import { connectToDatabase } from '@/service/mongo';
export type UrlFetchResponse = FetchResultItem[]; export type UrlFetchResponse = FetchResultItem[];
const fetchContent = async (req: NextApiRequest, res: NextApiResponse) => { const fetchContent = async (req: NextApiRequest, res: NextApiResponse) => {
try { try {
await connectToDatabase();
let { urlList = [] } = req.body as { urlList: string[] }; let { urlList = [] } = req.body as { urlList: string[] };
if (!urlList || urlList.length === 0) { if (!urlList || urlList.length === 0) {
......
projects/app/src/pages/api/plusApi/[...path].ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { request } from '@/service/common/api/request'; import { request } from '@fastgpt/common/plusApi/request';
import type { Method } from 'axios'; import type { Method } from 'axios';
import { connectToDatabase } from '@/service/mongo'; import { connectToDatabase } from '@/service/mongo';
......
projects/app/src/pages/api/support/openapi/delKey.ts
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction // Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase, OpenApi } from '@/service/mongo'; import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { MongoOpenApi } from '@fastgpt/support/openapi/schema';
import { authUser } from '@fastgpt/support/user/auth';
export default async function handler(req: NextApiRequest, res: NextApiResponse) { export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try { try {
await connectToDatabase();
const { id } = req.query as { id: string }; const { id } = req.query as { id: string };
if (!id) { if (!id) {
...@@ -14,9 +16,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) ...@@ -14,9 +16,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
await connectToDatabase(); await MongoOpenApi.findOneAndRemove({ _id: id, userId });
await OpenApi.findOneAndRemove({ _id: id, userId });
jsonRes(res); jsonRes(res);
} catch (err) { } catch (err) {
......
projects/app/src/pages/api/support/openapi/getKeys.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase, OpenApi } from '@/service/mongo'; import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { MongoOpenApi } from '@fastgpt/support/openapi/schema';
import { authUser } from '@fastgpt/support/user/auth';
import type { GetApiKeyProps } from '@/global/support/api/openapiReq.d'; import type { GetApiKeyProps } from '@/global/support/api/openapiReq.d';
export default async function handler(req: NextApiRequest, res: NextApiResponse) { export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try { try {
await connectToDatabase();
const { appId } = req.query as GetApiKeyProps; const { appId } = req.query as GetApiKeyProps;
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
await connectToDatabase(); const findResponse = await MongoOpenApi.find({ userId, appId }).sort({ _id: -1 });
const findResponse = await OpenApi.find({ userId, appId }).sort({ _id: -1 });
jsonRes(res, { jsonRes(res, {
data: findResponse.map((item) => item.toObject()) data: findResponse.map((item) => item.toObject())
......
projects/app/src/pages/api/support/openapi/postKey.ts
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction // Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase, OpenApi } from '@/service/mongo'; import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { MongoOpenApi } from '@fastgpt/support/openapi/schema';
import { authUser } from '@fastgpt/support/user/auth';
import { customAlphabet } from 'nanoid'; import { customAlphabet } from 'nanoid';
import type { EditApiKeyProps } from '@/global/support/api/openapiReq.d'; import type { EditApiKeyProps } from '@/global/support/api/openapiReq.d';
export default async function handler(req: NextApiRequest, res: NextApiResponse) { export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try { try {
await connectToDatabase();
const { appId, name, limit } = req.body as EditApiKeyProps; const { appId, name, limit } = req.body as EditApiKeyProps;
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
await connectToDatabase(); const count = await MongoOpenApi.find({ userId, appId }).countDocuments();
const count = await OpenApi.find({ userId, appId }).countDocuments();
if (count >= 10) { if (count >= 10) {
throw new Error('最多 10 组 API 秘钥'); throw new Error('最多 10 组 API 秘钥');
...@@ -25,7 +25,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) ...@@ -25,7 +25,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
); );
const apiKey = `${global.systemEnv?.openapiPrefix || 'fastgpt'}-${nanoid()}`; const apiKey = `${global.systemEnv?.openapiPrefix || 'fastgpt'}-${nanoid()}`;
await OpenApi.create({ await MongoOpenApi.create({
userId, userId,
apiKey, apiKey,
appId, appId,
......
projects/app/src/pages/api/support/openapi/putKey.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase, OpenApi } from '@/service/mongo'; import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { MongoOpenApi } from '@fastgpt/support/openapi/schema';
import { authUser } from '@fastgpt/support/user/auth';
import type { EditApiKeyProps } from '@/global/support/api/openapiReq.d'; import type { EditApiKeyProps } from '@/global/support/api/openapiReq.d';
export default async function handler(req: NextApiRequest, res: NextApiResponse) { export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try { try {
await connectToDatabase();
const { _id, name, limit } = req.body as EditApiKeyProps & { _id: string }; const { _id, name, limit } = req.body as EditApiKeyProps & { _id: string };
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
await connectToDatabase(); await MongoOpenApi.findOneAndUpdate(
await OpenApi.findOneAndUpdate(
{ {
_id, _id,
userId userId
......
projects/app/src/pages/api/support/outLink/create.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase, OutLink } from '@/service/mongo'; import { connectToDatabase } from '@/service/mongo';
import { authApp, authUser } from '@/service/utils/auth'; import { MongoOutLink } from '@fastgpt/support/outLink/schema';
import type { OutLinkEditType } from '@/types/support/outLink'; import { authApp } from '@/service/utils/auth';
import { authUser } from '@fastgpt/support/user/auth';
import type { OutLinkEditType } from '@fastgpt/support/outLink/type.d';
import { customAlphabet } from 'nanoid'; import { customAlphabet } from 'nanoid';
import { OutLinkTypeEnum } from '@/constants/chat'; import { OutLinkTypeEnum } from '@fastgpt/support/outLink/constant';
const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 24); const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 24);
/* create a shareChat */ /* create a shareChat */
export default async function handler(req: NextApiRequest, res: NextApiResponse) { export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try { try {
await connectToDatabase();
const { appId, ...props } = req.body as OutLinkEditType & { const { appId, ...props } = req.body as OutLinkEditType & {
appId: string; appId: string;
type: `${OutLinkTypeEnum}`; type: `${OutLinkTypeEnum}`;
}; };
await connectToDatabase();
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
await authApp({ await authApp({
appId, appId,
...@@ -25,7 +26,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) ...@@ -25,7 +26,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
}); });
const shareId = nanoid(); const shareId = nanoid();
await OutLink.create({ await MongoOutLink.create({
shareId, shareId,
userId, userId,
appId, appId,
......
projects/app/src/pages/api/support/outLink/delete.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase, OutLink } from '@/service/mongo'; import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { MongoOutLink } from '@fastgpt/support/outLink/schema';
import { authUser } from '@fastgpt/support/user/auth';
/* delete a shareChat by shareChatId */ /* delete a shareChat by shareChatId */
export default async function handler(req: NextApiRequest, res: NextApiResponse) { export default async function handler(req: NextApiRequest, res: NextApiResponse) {
...@@ -14,7 +15,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) ...@@ -14,7 +15,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
await OutLink.findOneAndRemove({ await MongoOutLink.findOneAndRemove({
_id: id, _id: id,
userId userId
}); });
......
projects/app/src/pages/api/support/outLink/init.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase, OutLink, User } from '@/service/mongo'; import { connectToDatabase } from '@/service/mongo';
import { MongoOutLink } from '@fastgpt/support/outLink/schema';
import { MongoUser } from '@fastgpt/support/user/schema';
import type { InitShareChatResponse } from '@/global/support/api/outLinkRes.d'; import type { InitShareChatResponse } from '@/global/support/api/outLinkRes.d';
import { authApp } from '@/service/utils/auth'; import { authApp } from '@/service/utils/auth';
import { HUMAN_ICON } from '@/constants/chat'; import { HUMAN_ICON } from '@/constants/chat';
import { getChatModelNameList, getGuideModule } from '@/components/ChatBox/utils'; import { getChatModelNameList, getGuideModule } from '@/components/ChatBox/utils';
import { authShareChatInit } from '@/service/support/outLink/auth'; import { authShareChatInit } from '@fastgpt/support/outLink/auth';
/* init share chat window */ /* init share chat window */
export default async function handler(req: NextApiRequest, res: NextApiResponse) { export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try { try {
await connectToDatabase();
let { shareId, authToken } = req.query as { let { shareId, authToken } = req.query as {
shareId: string; shareId: string;
authToken?: string; authToken?: string;
...@@ -19,10 +22,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) ...@@ -19,10 +22,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
throw new Error('params is error'); throw new Error('params is error');
} }
await connectToDatabase();
// get shareChat // get shareChat
const shareChat = await OutLink.findOne({ shareId }); const shareChat = await MongoOutLink.findOne({ shareId });
if (!shareChat) { if (!shareChat) {
return jsonRes(res, { return jsonRes(res, {
...@@ -38,8 +39,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) ...@@ -38,8 +39,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
userId: String(shareChat.userId), userId: String(shareChat.userId),
authOwner: false authOwner: false
}), }),
User.findById(shareChat.userId, 'avatar'), MongoUser.findById(shareChat.userId, 'avatar'),
authShareChatInit(authToken, shareChat.limit?.hookUrl) authShareChatInit({
authToken,
tokenUrl: shareChat.limit?.hookUrl
})
]); ]);
jsonRes<InitShareChatResponse>(res, { jsonRes<InitShareChatResponse>(res, {
......
projects/app/src/pages/api/support/outLink/list.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase, OutLink } from '@/service/mongo'; import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { MongoOutLink } from '@fastgpt/support/outLink/schema';
import { hashPassword } from '@/service/utils/tools'; import { authUser } from '@fastgpt/support/user/auth';
/* get shareChat list by appId */ /* get shareChat list by appId */
export default async function handler(req: NextApiRequest, res: NextApiResponse) { export default async function handler(req: NextApiRequest, res: NextApiResponse) {
...@@ -15,7 +15,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) ...@@ -15,7 +15,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
const data = await OutLink.find({ const data = await MongoOutLink.find({
appId, appId,
userId userId
}).sort({ }).sort({
......
projects/app/src/pages/api/support/outLink/update.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase, OutLink } from '@/service/mongo'; import { connectToDatabase } from '@/service/mongo';
import type { OutLinkEditType } from '@/types/support/outLink'; import { MongoOutLink } from '@fastgpt/support/outLink/schema';
import type { OutLinkEditType } from '@fastgpt/support/outLink/type.d';
export default async function handler(req: NextApiRequest, res: NextApiResponse) { export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try { try {
...@@ -9,7 +10,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) ...@@ -9,7 +10,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
const { _id, name, responseDetail, limit } = req.body as OutLinkEditType & {}; const { _id, name, responseDetail, limit } = req.body as OutLinkEditType & {};
await OutLink.findByIdAndUpdate(_id, { await MongoOutLink.findByIdAndUpdate(_id, {
name, name,
responseDetail, responseDetail,
limit limit
......
projects/app/src/pages/api/system/file/delete.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo'; import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import { GridFSStorage } from '@/service/lib/gridfs'; import { GridFSStorage } from '@/service/lib/gridfs';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
......
projects/app/src/pages/api/system/file/readUrl.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo'; import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import jwt from 'jsonwebtoken'; import jwt from 'jsonwebtoken';
import { ERROR_ENUM } from '@/service/errorCode'; import { ERROR_ENUM } from '@fastgpt/common/constant/errorCode';
import { GridFSStorage } from '@/service/lib/gridfs'; import { GridFSStorage } from '@/service/lib/gridfs';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
......
projects/app/src/pages/api/system/file/upload.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo'; import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import { GridFSStorage } from '@/service/lib/gridfs'; import { GridFSStorage } from '@/service/lib/gridfs';
import { customAlphabet } from 'nanoid'; import { customAlphabet } from 'nanoid';
import multer from 'multer'; import multer from 'multer';
......
projects/app/src/pages/api/system/file/uploadImage.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase, Image } from '@/service/mongo'; import { connectToDatabase, Image } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
type Props = { base64Img: string }; type Props = { base64Img: string };
......
projects/app/src/pages/api/system/getInitData.ts
import type { FeConfigsType, SystemEnvType } from '@/types'; import type { SystemEnvType } from '@/types';
import type { FeConfigsType } from '@fastgpt/common/type/index.d';
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { readFileSync } from 'fs'; import { readFileSync } from 'fs';
import type { InitDateResponse } from '@/global/common/api/systemRes'; import type { InitDateResponse } from '@/global/common/api/systemRes';
import { type VectorModelItemType, FunctionModelItemType } from '@/types/model'; import type { VectorModelItemType, FunctionModelItemType } from '@/types/model';
import { formatPrice } from '@fastgpt/common/bill';
import { getTikTokenEnc } from '@/utils/common/tiktoken';
import { initHttpAgent } from '@fastgpt/core/init';
export default async function handler(req: NextApiRequest, res: NextApiResponse) { export default async function handler(req: NextApiRequest, res: NextApiResponse) {
if (!global.feConfigs) { getInitConfig();
await getInitConfig(); getModelPrice();
}
jsonRes<InitDateResponse>(res, { jsonRes<InitDateResponse>(res, {
data: { data: {
feConfigs: global.feConfigs, feConfigs: global.feConfigs,
chatModels: global.chatModels, chatModels: global.chatModels,
qaModel: global.qaModel, qaModel: global.qaModel,
vectorModels: global.vectorModels, vectorModels: global.vectorModels,
priceMd: global.priceMd,
systemVersion: global.systemVersion || '0.0.0' systemVersion: global.systemVersion || '0.0.0'
} }
}); });
...@@ -23,7 +28,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) ...@@ -23,7 +28,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
const defaultSystemEnv: SystemEnvType = { const defaultSystemEnv: SystemEnvType = {
vectorMaxProcess: 15, vectorMaxProcess: 15,
qaMaxProcess: 15, qaMaxProcess: 15,
pgIvfflatProbe: 20 pgHNSWEfSearch: 40
}; };
const defaultFeConfigs: FeConfigsType = { const defaultFeConfigs: FeConfigsType = {
show_emptyChat: true, show_emptyChat: true,
...@@ -104,7 +109,17 @@ const defaultVectorModels: VectorModelItemType[] = [ ...@@ -104,7 +109,17 @@ const defaultVectorModels: VectorModelItemType[] = [
} }
]; ];
export async function getInitConfig() { export function initGlobal() {
// init tikToken
getTikTokenEnc();
initHttpAgent();
global.qaQueueLen = 0;
global.vectorQueueLen = 0;
global.sendInformQueue = [];
global.sendInformQueueLen = 0;
}
export function getInitConfig() {
try { try {
if (global.feConfigs) return; if (global.feConfigs) return;
...@@ -143,6 +158,7 @@ export function setDefaultData() { ...@@ -143,6 +158,7 @@ export function setDefaultData() {
global.extractModel = defaultExtractModel; global.extractModel = defaultExtractModel;
global.cqModel = defaultCQModel; global.cqModel = defaultCQModel;
global.qgModel = defaultQGModel; global.qgModel = defaultQGModel;
global.priceMd = '';
} }
export function getSystemVersion() { export function getSystemVersion() {
...@@ -160,3 +176,21 @@ export function getSystemVersion() { ...@@ -160,3 +176,21 @@ export function getSystemVersion() {
global.systemVersion = '0.0.0'; global.systemVersion = '0.0.0';
} }
} }
function getModelPrice() {
if (global.priceMd) return;
global.priceMd = `| 计费项 | 价格: 元/ 1K tokens(包含上下文)|
| --- | --- |
${global.vectorModels
?.map((item) => `| 索引-${item.name} | ${formatPrice(item.price, 1000)} |`)
.join('\n')}
${global.chatModels
?.map((item) => `| 对话-${item.name} | ${formatPrice(item.price, 1000)} |`)
.join('\n')}
| 文件QA拆分 | ${formatPrice(global.qaModel?.price, 1000)} |
| 高级编排 - 问题分类 | ${formatPrice(global.cqModel?.price, 1000)} |
| 高级编排 - 内容提取 | ${formatPrice(global.extractModel?.price, 1000)} |
| 下一步指引 | ${formatPrice(global.qgModel?.price, 1000)} |
`;
console.log(global.priceMd);
}
projects/app/src/pages/api/user/account/loginByPassword.ts
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction // Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { MongoUser } from '@fastgpt/support/user/schema';
import { setCookie } from '@/service/utils/tools';
import { generateToken } from '@fastgpt/support/user/tools';
import { connectToDatabase } from '@/service/mongo'; import { connectToDatabase } from '@/service/mongo';
import { User } from '@/service/models/user';
import { generateToken, setCookie } from '@/service/utils/tools';
export default async function handler(req: NextApiRequest, res: NextApiResponse) { export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try { try {
await connectToDatabase();
const { username, password } = req.body; const { username, password } = req.body;
if (!username || !password) { if (!username || !password) {
throw new Error('缺少参数'); throw new Error('缺少参数');
} }
await connectToDatabase();
// 检测用户是否存在 // 检测用户是否存在
const authUser = await User.findOne({ const authUser = await MongoUser.findOne({
username username
}); });
if (!authUser) { if (!authUser) {
throw new Error('用户未注册'); throw new Error('用户未注册');
} }
const user = await User.findOne({ const user = await MongoUser.findOne({
username, username,
password password
}); });
......
projects/app/src/pages/api/user/account/paySuccess.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { TrainingData } from '@/service/mongo'; import { TrainingData, connectToDatabase } from '@/service/mongo';
import { startQueue } from '@/service/utils/tools'; import { startQueue } from '@/service/utils/tools';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try { try {
await connectToDatabase();
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
await unlockTask(userId); await unlockTask(userId);
} catch (error) {} } catch (error) {}
......
projects/app/src/pages/api/user/account/tokenLogin.ts
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction // Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { authUser } from '@fastgpt/support/user/auth';
import { MongoUser } from '@fastgpt/support/user/schema';
import { connectToDatabase } from '@/service/mongo'; import { connectToDatabase } from '@/service/mongo';
import { User } from '@/service/models/user';
import { authUser } from '@/service/utils/auth';
export default async function handler(req: NextApiRequest, res: NextApiResponse) { export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try { try {
const { userId } = await authUser({ req, authToken: true });
await connectToDatabase(); await connectToDatabase();
const { userId } = await authUser({ req, authToken: true });
// 根据 id 获取用户信息 // 根据 id 获取用户信息
const user = await User.findById(userId); const user = await MongoUser.findById(userId);
if (!user) { if (!user) {
throw new Error('账号异常'); throw new Error('账号异常');
......
projects/app/src/pages/api/user/account/update.ts
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction // Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { User } from '@/service/models/user'; import { MongoUser } from '@fastgpt/support/user/schema';
import { connectToDatabase } from '@/service/mongo'; import { authUser } from '@fastgpt/support/user/auth';
import { authUser } from '@/service/utils/auth';
import { UserUpdateParams } from '@/types/user'; import { UserUpdateParams } from '@/types/user';
import { getAIApi, openaiBaseUrl } from '@fastgpt/core/ai/config'; import { getAIApi, openaiBaseUrl } from '@fastgpt/core/ai/config';
import { connectToDatabase } from '@/service/mongo';
/* update user info */ /* update user info */
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try { try {
await connectToDatabase();
const { avatar, timezone, openaiAccount } = req.body as UserUpdateParams; const { avatar, timezone, openaiAccount } = req.body as UserUpdateParams;
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
// auth key // auth key
if (openaiAccount?.key) { if (openaiAccount?.key) {
console.log('auth user openai key', openaiAccount?.key); console.log('auth user openai key', openaiAccount?.key);
...@@ -35,7 +34,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -35,7 +34,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
} }
// 更新对应的记录 // 更新对应的记录
await User.updateOne( await MongoUser.updateOne(
{ {
_id: userId _id: userId
}, },
......
projects/app/src/pages/api/user/account/updatePasswordByOld.ts
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction // Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { User } from '@/service/models/user'; import { authUser } from '@fastgpt/support/user/auth';
import { MongoUser } from '@fastgpt/support/user/schema';
import { connectToDatabase } from '@/service/mongo'; import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) { export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try { try {
await connectToDatabase();
const { oldPsw, newPsw } = req.body as { oldPsw: string; newPsw: string }; const { oldPsw, newPsw } = req.body as { oldPsw: string; newPsw: string };
if (!oldPsw || !newPsw) { if (!oldPsw || !newPsw) {
throw new Error('Params is missing'); throw new Error('Params is missing');
} }
await connectToDatabase();
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
// auth old password // auth old password
const user = await User.findOne({ const user = await MongoUser.findOne({
_id: userId, _id: userId,
password: oldPsw password: oldPsw
}); });
...@@ -28,7 +27,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...@@ -28,7 +27,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
} }
// 更新对应的记录 // 更新对应的记录
await User.findByIdAndUpdate(userId, { await MongoUser.findByIdAndUpdate(userId, {
password: newPsw password: newPsw
}); });
......
projects/app/src/pages/api/user/getBill.ts
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction // Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase, Bill } from '@/service/mongo'; import { Bill, connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import { adaptBill } from '@/utils/adapt'; import { adaptBill } from '@/utils/adapt';
import { addDays } from 'date-fns'; import { addDays } from 'date-fns';
export default async function handler(req: NextApiRequest, res: NextApiResponse) { export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try { try {
await connectToDatabase();
const { const {
pageNum = 1, pageNum = 1,
pageSize = 10, pageSize = 10,
...@@ -22,8 +23,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) ...@@ -22,8 +23,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
const where = { const where = {
userId, userId,
time: { time: {
......
projects/app/src/pages/api/user/getPayOrders.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import { connectToDatabase, Pay } from '@/service/mongo'; import { Pay, connectToDatabase } from '@/service/mongo';
export default async function handler(req: NextApiRequest, res: NextApiResponse) { export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try { try {
const { userId } = await authUser({ req, authToken: true });
await connectToDatabase(); await connectToDatabase();
const { userId } = await authUser({ req, authToken: true });
const records = await Pay.find({ const records = await Pay.find({
userId, userId,
......
projects/app/src/pages/api/user/inform/countUnread.ts
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction // Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase, Inform } from '@/service/mongo'; import { Inform, connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
export default async function handler(req: NextApiRequest, res: NextApiResponse) { export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try { try {
await connectToDatabase();
if (!req.headers.cookie) { if (!req.headers.cookie) {
return jsonRes(res, { return jsonRes(res, {
data: 0 data: 0
...@@ -13,8 +14,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) ...@@ -13,8 +14,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
} }
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
const data = await Inform.countDocuments({ const data = await Inform.countDocuments({
userId, userId,
read: false read: false
......
projects/app/src/pages/api/user/inform/list.ts
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction // Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase, Inform } from '@/service/mongo'; import { Inform, connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
export default async function handler(req: NextApiRequest, res: NextApiResponse) { export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try { try {
await connectToDatabase();
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
const { pageNum, pageSize = 10 } = req.body as { const { pageNum, pageSize = 10 } = req.body as {
...@@ -13,8 +14,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) ...@@ -13,8 +14,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
pageSize: number; pageSize: number;
}; };
await connectToDatabase();
const [informs, total] = await Promise.all([ const [informs, total] = await Promise.all([
Inform.find({ userId }) Inform.find({ userId })
.sort({ time: -1 }) // 按照创建时间倒序排列 .sort({ time: -1 }) // 按照创建时间倒序排列
......
projects/app/src/pages/api/user/inform/read.ts
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction // Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase, Inform } from '@/service/mongo'; import { Inform, connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
export default async function handler(req: NextApiRequest, res: NextApiResponse) { export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try { try {
const { userId } = await authUser({ req, authToken: true });
await connectToDatabase(); await connectToDatabase();
const { userId } = await authUser({ req, authToken: true });
const { id } = req.query as { id: string }; const { id } = req.query as { id: string };
......
projects/app/src/pages/api/user/inform/send.ts
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction // Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase, Inform, User } from '@/service/mongo'; import { Inform, connectToDatabase } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import { InformTypeEnum } from '@/constants/user'; import { InformTypeEnum } from '@/constants/user';
import { startSendInform } from '@/service/events/sendInform'; import { startSendInform } from '@/service/events/sendInform';
import { MongoUser } from '@fastgpt/support/user/schema';
export type Props = { export type Props = {
type: `${InformTypeEnum}`; type: `${InformTypeEnum}`;
...@@ -15,9 +16,8 @@ export type Props = { ...@@ -15,9 +16,8 @@ export type Props = {
export default async function handler(req: NextApiRequest, res: NextApiResponse) { export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try { try {
await authUser({ req, authRoot: true });
await connectToDatabase(); await connectToDatabase();
await authUser({ req, authRoot: true });
jsonRes(res, { jsonRes(res, {
data: await sendInform(req.body), data: await sendInform(req.body),
...@@ -62,7 +62,7 @@ export async function sendInform({ type, title, content, userId }: Props) { ...@@ -62,7 +62,7 @@ export async function sendInform({ type, title, content, userId }: Props) {
} }
// send to all user // send to all user
const users = await User.find({}, '_id'); const users = await MongoUser.find({}, '_id');
await Inform.insertMany( await Inform.insertMany(
users.map(({ _id }) => ({ users.map(({ _id }) => ({
type, type,
......
projects/app/src/pages/api/user/promotion/getPromotionData.ts
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction // Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase, User, promotionRecord } from '@/service/mongo'; import { connectToDatabase, promotionRecord } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import mongoose from 'mongoose'; import mongoose from '@fastgpt/common/mongo';
import { MongoUser } from '@fastgpt/support/user/schema';
export default async function handler(req: NextApiRequest, res: NextApiResponse) { export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try { try {
await connectToDatabase(); await connectToDatabase();
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
const invitedAmount = await User.countDocuments({ const invitedAmount = await MongoUser.countDocuments({
inviterId: userId inviterId: userId
}); });
......
projects/app/src/pages/api/user/promotion/getPromotions.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { connectToDatabase, promotionRecord } from '@/service/mongo'; import { connectToDatabase, promotionRecord } from '@/service/mongo';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
export default async function handler(req: NextApiRequest, res: NextApiResponse) { export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try { try {
await connectToDatabase();
let { pageNum = 1, pageSize = 10 } = req.body as { let { pageNum = 1, pageSize = 10 } = req.body as {
pageNum: number; pageNum: number;
pageSize: number; pageSize: number;
...@@ -12,8 +13,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) ...@@ -12,8 +13,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
const { userId } = await authUser({ req, authToken: true }); const { userId } = await authUser({ req, authToken: true });
await connectToDatabase();
const data = await promotionRecord const data = await promotionRecord
.find( .find(
{ {
......
projects/app/src/pages/api/v1/chat/completions.ts
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { connectToDatabase } from '@/service/mongo'; import { authApp } from '@/service/utils/auth';
import { authUser, authApp, AuthUserTypeEnum } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import { AuthUserTypeEnum } from '@fastgpt/support/user/auth';
import { sseErrRes, jsonRes } from '@/service/response'; import { sseErrRes, jsonRes } from '@/service/response';
import { addLog, withNextCors } from '@/service/utils/tools'; import { addLog, withNextCors } from '@/service/utils/tools';
import { ChatRoleEnum, ChatSourceEnum, sseResponseEventEnum } from '@/constants/chat'; import { ChatRoleEnum, ChatSourceEnum, sseResponseEventEnum } from '@/constants/chat';
...@@ -27,16 +28,17 @@ import { AppModuleItemType, RunningModuleItemType } from '@/types/app'; ...@@ -27,16 +28,17 @@ import { AppModuleItemType, RunningModuleItemType } from '@/types/app';
import { pushChatBill } from '@/service/common/bill/push'; import { pushChatBill } from '@/service/common/bill/push';
import { BillSourceEnum } from '@/constants/user'; import { BillSourceEnum } from '@/constants/user';
import { ChatHistoryItemResType } from '@/types/chat'; import { ChatHistoryItemResType } from '@/types/chat';
import { UserModelSchema } from '@/types/mongoSchema'; import type { UserModelSchema } from '@fastgpt/support/user/type.d';
import { SystemInputEnum } from '@/constants/app'; import { SystemInputEnum } from '@/constants/app';
import { getSystemTime } from '@/utils/user'; import { getSystemTime } from '@/utils/user';
import { authOutLinkChat } from '@/service/support/outLink/auth'; import { authOutLinkChat } from '@fastgpt/support/outLink/auth';
import { pushResult2Remote, updateOutLinkUsage } from '@fastgpt/support/outLink/tools';
import requestIp from 'request-ip'; import requestIp from 'request-ip';
import { replaceVariable } from '@/utils/common/tools/text'; import { replaceVariable } from '@/utils/common/tools/text';
import type { ModuleDispatchProps } from '@/types/core/chat/type'; import type { ModuleDispatchProps } from '@/types/core/chat/type';
import { selectShareResponse } from '@/utils/service/core/chat'; import { selectShareResponse } from '@/utils/service/core/chat';
import { pushResult2Remote, updateOutLinkUsage } from '@/service/support/outLink'; import { updateApiKeyUsage } from '@fastgpt/support/openapi/tools';
import { updateApiKeyUsage } from '@/service/support/openapi'; import { connectToDatabase } from '@/service/mongo';
type FastGptWebChatProps = { type FastGptWebChatProps = {
chatId?: string; // undefined: nonuse history, '': new chat, 'xxxxx': use history chatId?: string; // undefined: nonuse history, '': new chat, 'xxxxx': use history
...@@ -80,6 +82,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex ...@@ -80,6 +82,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
} = req.body as Props; } = req.body as Props;
try { try {
await connectToDatabase();
// body data check // body data check
if (!messages) { if (!messages) {
throw new Error('Prams Error'); throw new Error('Prams Error');
...@@ -91,7 +94,6 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex ...@@ -91,7 +94,6 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
throw new Error('messages is empty'); throw new Error('messages is empty');
} }
await connectToDatabase();
let startTime = Date.now(); let startTime = Date.now();
/* user auth */ /* user auth */
......
projects/app/src/pages/api/v1/chat/getHistory.ts
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction // Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next'; import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response'; import { jsonRes } from '@/service/response';
import { authUser } from '@/service/utils/auth'; import { authUser } from '@fastgpt/support/user/auth';
import { connectToDatabase, ChatItem } from '@/service/mongo'; import { ChatItem, connectToDatabase } from '@/service/mongo';
import { Types } from 'mongoose'; import { Types } from '@fastgpt/common/mongo';
import type { ChatItemType } from '@/types/chat'; import type { ChatItemType } from '@/types/chat';
export type Props = { export type Props = {
......
projects/app/src/pages/app/detail/components/DatasetSelectModal.tsx
...@@ -20,7 +20,7 @@ import MySlider from '@/components/Slider'; ...@@ -20,7 +20,7 @@ import MySlider from '@/components/Slider';
import MyTooltip from '@/components/MyTooltip'; import MyTooltip from '@/components/MyTooltip';
import MyModal from '@/components/MyModal'; import MyModal from '@/components/MyModal';
import MyIcon from '@/components/Icon'; import MyIcon from '@/components/Icon';
import { KbTypeEnum } from '@/constants/dataset'; import { DatasetTypeEnum } from '@fastgpt/core/dataset/constant';
import { useTranslation } from 'react-i18next'; import { useTranslation } from 'react-i18next';
import { useQuery } from '@tanstack/react-query'; import { useQuery } from '@tanstack/react-query';
import { useDatasetStore } from '@/web/core/store/dataset'; import { useDatasetStore } from '@/web/core/store/dataset';
...@@ -116,7 +116,7 @@ export const DatasetSelectModal = ({ ...@@ -116,7 +116,7 @@ export const DatasetSelectModal = ({
<MyTooltip <MyTooltip
key={item._id} key={item._id}
label={ label={
item.type === KbTypeEnum.dataset item.type === DatasetTypeEnum.dataset
? t('kb.Select Dataset') ? t('kb.Select Dataset')
: t('kb.Select Folder') : t('kb.Select Folder')
} }
...@@ -131,9 +131,9 @@ export const DatasetSelectModal = ({ ...@@ -131,9 +131,9 @@ export const DatasetSelectModal = ({
boxShadow: 'md' boxShadow: 'md'
}} }}
onClick={() => { onClick={() => {
if (item.type === KbTypeEnum.folder) { if (item.type === DatasetTypeEnum.folder) {
setParentId(item._id); setParentId(item._id);
} else if (item.type === KbTypeEnum.dataset) { } else if (item.type === DatasetTypeEnum.dataset) {
const vectorModel = selectedKbList[0]?.vectorModel?.model; const vectorModel = selectedKbList[0]?.vectorModel?.model;
if (vectorModel && vectorModel !== item.vectorModel.model) { if (vectorModel && vectorModel !== item.vectorModel.model) {
...@@ -161,7 +161,7 @@ export const DatasetSelectModal = ({ ...@@ -161,7 +161,7 @@ export const DatasetSelectModal = ({
</Box> </Box>
</Flex> </Flex>
<Flex justifyContent={'flex-end'} alignItems={'center'} fontSize={'sm'}> <Flex justifyContent={'flex-end'} alignItems={'center'} fontSize={'sm'}>
{item.type === KbTypeEnum.folder ? ( {item.type === DatasetTypeEnum.folder ? (
<Box color={'myGray.500'}>{t('Folder')}</Box> <Box color={'myGray.500'}>{t('Folder')}</Box>
) : ( ) : (
<> <>
......
projects/app/src/pages/app/detail/components/OutLink/Share.tsx
...@@ -34,10 +34,10 @@ import { formatTimeToChatTime } from '@/utils/tools'; ...@@ -34,10 +34,10 @@ import { formatTimeToChatTime } from '@/utils/tools';
import { useCopyData } from '@/web/common/hooks/useCopyData'; import { useCopyData } from '@/web/common/hooks/useCopyData';
import { useForm } from 'react-hook-form'; import { useForm } from 'react-hook-form';
import { defaultOutLinkForm } from '@/constants/model'; import { defaultOutLinkForm } from '@/constants/model';
import type { OutLinkEditType } from '@/types/support/outLink'; import type { OutLinkEditType } from '@fastgpt/support/outLink/type.d';
import { useRequest } from '@/web/common/hooks/useRequest'; import { useRequest } from '@/web/common/hooks/useRequest';
import { formatPrice } from '@fastgpt/common/bill/index'; import { formatPrice } from '@fastgpt/common/bill/index';
import { OutLinkTypeEnum } from '@/constants/chat'; import { OutLinkTypeEnum } from '@fastgpt/support/outLink/constant';
import { useTranslation } from 'react-i18next'; import { useTranslation } from 'react-i18next';
import { useToast } from '@/web/common/hooks/useToast'; import { useToast } from '@/web/common/hooks/useToast';
import { feConfigs } from '@/web/common/store/static'; import { feConfigs } from '@/web/common/store/static';
......
projects/app/src/pages/app/detail/components/OutLink/index.tsx
import React, { useState } from 'react'; import React, { useState } from 'react';
import { Box, useTheme } from '@chakra-ui/react'; import { Box, useTheme } from '@chakra-ui/react';
import { OutLinkTypeEnum } from '@/constants/chat'; import { OutLinkTypeEnum } from '@fastgpt/support/outLink/constant';
import dynamic from 'next/dynamic'; import dynamic from 'next/dynamic';
import MyRadio from '@/components/Radio'; import MyRadio from '@/components/Radio';
......
projects/app/src/pages/components/Navbar.tsx
...@@ -23,6 +23,16 @@ const Navbar = () => { ...@@ -23,6 +23,16 @@ const Navbar = () => {
...(feConfigs?.show_contact ...(feConfigs?.show_contact
? [ ? [
{ {
label: t('home.Commercial'),
key: 'community',
onClick: () => {
window.open(
'https://fael3z0zfze.feishu.cn/share/base/form/shrcnRxj3utrzjywsom96Px4sud',
'_blank'
);
}
},
{
label: t('home.Community'), label: t('home.Community'),
key: 'community', key: 'community',
onClick: () => { onClick: () => {
......
projects/app/src/pages/kb/detail/components/FileCard.tsx
...@@ -28,14 +28,13 @@ import MyInput from '@/components/MyInput'; ...@@ -28,14 +28,13 @@ import MyInput from '@/components/MyInput';
import dayjs from 'dayjs'; import dayjs from 'dayjs';
import { useRequest } from '@/web/common/hooks/useRequest'; import { useRequest } from '@/web/common/hooks/useRequest';
import { useLoading } from '@/web/common/hooks/useLoading'; import { useLoading } from '@/web/common/hooks/useLoading';
import { FileStatusEnum } from '@/constants/dataset';
import { useRouter } from 'next/router'; import { useRouter } from 'next/router';
import { usePagination } from '@/web/common/hooks/usePagination'; import { usePagination } from '@/web/common/hooks/usePagination';
import type { DatasetFileItemType } from '@/types/core/dataset/file'; import type { DatasetFileItemType } from '@/types/core/dataset/file';
import { useGlobalStore } from '@/web/common/store/global'; import { useGlobalStore } from '@/web/common/store/global';
import MyMenu from '@/components/MyMenu'; import MyMenu from '@/components/MyMenu';
import { useEditTitle } from '@/web/common/hooks/useEditTitle'; import { useEditTitle } from '@/web/common/hooks/useEditTitle';
import { datasetSpecialIds } from '@fastgpt/core/dataset/constant'; import { datasetSpecialIds, FileStatusEnum } from '@fastgpt/core/dataset/constant';
import { getFileIcon, getSpecialFileIcon } from '@fastgpt/common/tools/file'; import { getFileIcon, getSpecialFileIcon } from '@fastgpt/common/tools/file';
const FileCard = ({ kbId }: { kbId: string }) => { const FileCard = ({ kbId }: { kbId: string }) => {
......
projects/app/src/pages/kb/detail/components/Import/FileSelect.tsx
...@@ -2,7 +2,8 @@ import MyIcon from '@/components/Icon'; ...@@ -2,7 +2,8 @@ import MyIcon from '@/components/Icon';
import { useLoading } from '@/web/common/hooks/useLoading'; import { useLoading } from '@/web/common/hooks/useLoading';
import { useSelectFile } from '@/web/common/hooks/useSelectFile'; import { useSelectFile } from '@/web/common/hooks/useSelectFile';
import { useToast } from '@/web/common/hooks/useToast'; import { useToast } from '@/web/common/hooks/useToast';
import { simpleText, splitText2Chunks } from '@/utils/file'; import { splitText2Chunks } from '@/utils/file';
import { simpleText } from '@fastgpt/common/tools/str';
import { import {
uploadFiles, uploadFiles,
fileDownload, fileDownload,
......
projects/app/src/pages/kb/list/component/EditFolderModal.tsx
...@@ -4,7 +4,7 @@ import MyModal from '@/components/MyModal'; ...@@ -4,7 +4,7 @@ import MyModal from '@/components/MyModal';
import { useTranslation } from 'react-i18next'; import { useTranslation } from 'react-i18next';
import { useRequest } from '@/web/common/hooks/useRequest'; import { useRequest } from '@/web/common/hooks/useRequest';
import { postCreateDataset, putDatasetById } from '@/web/core/api/dataset'; import { postCreateDataset, putDatasetById } from '@/web/core/api/dataset';
import { FolderAvatarSrc, KbTypeEnum } from '@/constants/dataset'; import { FolderAvatarSrc, DatasetTypeEnum } from '@fastgpt/core/dataset/constant';
const EditFolderModal = ({ const EditFolderModal = ({
onClose, onClose,
...@@ -47,7 +47,7 @@ const EditFolderModal = ({ ...@@ -47,7 +47,7 @@ const EditFolderModal = ({
return postCreateDataset({ return postCreateDataset({
parentId, parentId,
name: val, name: val,
type: KbTypeEnum.folder, type: DatasetTypeEnum.folder,
avatar: FolderAvatarSrc, avatar: FolderAvatarSrc,
tags: [] tags: []
}); });
......
projects/app/src/pages/kb/list/component/MoveModal.tsx
...@@ -14,7 +14,7 @@ import Avatar from '@/components/Avatar'; ...@@ -14,7 +14,7 @@ import Avatar from '@/components/Avatar';
import MyTooltip from '@/components/MyTooltip'; import MyTooltip from '@/components/MyTooltip';
import MyModal from '@/components/MyModal'; import MyModal from '@/components/MyModal';
import MyIcon from '@/components/Icon'; import MyIcon from '@/components/Icon';
import { KbTypeEnum } from '@/constants/dataset'; import { DatasetTypeEnum } from '@fastgpt/core/dataset/constant';
import { useTranslation } from 'react-i18next'; import { useTranslation } from 'react-i18next';
import { useQuery } from '@tanstack/react-query'; import { useQuery } from '@tanstack/react-query';
import { getDatasets, putDatasetById, getDatasetPaths } from '@/web/core/api/dataset'; import { getDatasets, putDatasetById, getDatasetPaths } from '@/web/core/api/dataset';
...@@ -112,7 +112,7 @@ const MoveModal = ({ ...@@ -112,7 +112,7 @@ const MoveModal = ({
<MyTooltip <MyTooltip
key={item._id} key={item._id}
label={ label={
item.type === KbTypeEnum.dataset item.type === DatasetTypeEnum.dataset
? t('kb.Select Dataset') ? t('kb.Select Dataset')
: t('kb.Select Folder') : t('kb.Select Folder')
} }
...@@ -142,7 +142,7 @@ const MoveModal = ({ ...@@ -142,7 +142,7 @@ const MoveModal = ({
</Box> </Box>
</Flex> </Flex>
<Flex justifyContent={'flex-end'} alignItems={'center'} fontSize={'sm'}> <Flex justifyContent={'flex-end'} alignItems={'center'} fontSize={'sm'}>
{item.type === KbTypeEnum.folder ? ( {item.type === DatasetTypeEnum.folder ? (
<Box color={'myGray.500'}>{t('Folder')}</Box> <Box color={'myGray.500'}>{t('Folder')}</Box>
) : ( ) : (
<> <>
......
projects/app/src/pages/kb/list/index.tsx
...@@ -26,7 +26,7 @@ import Avatar from '@/components/Avatar'; ...@@ -26,7 +26,7 @@ import Avatar from '@/components/Avatar';
import MyIcon from '@/components/Icon'; import MyIcon from '@/components/Icon';
import { serviceSideProps } from '@/web/common/utils/i18n'; import { serviceSideProps } from '@/web/common/utils/i18n';
import dynamic from 'next/dynamic'; import dynamic from 'next/dynamic';
import { FolderAvatarSrc, KbTypeEnum } from '@/constants/dataset'; import { FolderAvatarSrc, DatasetTypeEnum } from '@fastgpt/core/dataset/constant';
import Tag from '@/components/Tag'; import Tag from '@/components/Tag';
import MyMenu from '@/components/MyMenu'; import MyMenu from '@/components/MyMenu';
import { useRequest } from '@/web/common/hooks/useRequest'; import { useRequest } from '@/web/common/hooks/useRequest';
...@@ -46,8 +46,8 @@ const Kb = () => { ...@@ -46,8 +46,8 @@ const Kb = () => {
const { setLoading } = useGlobalStore(); const { setLoading } = useGlobalStore();
const DeleteTipsMap = useRef({ const DeleteTipsMap = useRef({
[KbTypeEnum.folder]: t('kb.deleteFolderTips'), [DatasetTypeEnum.folder]: t('kb.deleteFolderTips'),
[KbTypeEnum.dataset]: t('kb.deleteDatasetTips') [DatasetTypeEnum.dataset]: t('kb.deleteDatasetTips')
}); });
const { openConfirm, ConfirmModal } = useConfirm({ const { openConfirm, ConfirmModal } = useConfirm({
...@@ -224,7 +224,7 @@ const Kb = () => { ...@@ -224,7 +224,7 @@ const Kb = () => {
border={theme.borders.md} border={theme.borders.md}
boxShadow={'none'} boxShadow={'none'}
position={'relative'} position={'relative'}
data-drag-id={kb.type === KbTypeEnum.folder ? kb._id : undefined} data-drag-id={kb.type === DatasetTypeEnum.folder ? kb._id : undefined}
borderColor={dragTargetId === kb._id ? 'myBlue.600' : ''} borderColor={dragTargetId === kb._id ? 'myBlue.600' : ''}
draggable draggable
onDragStart={(e) => { onDragStart={(e) => {
...@@ -234,7 +234,7 @@ const Kb = () => { ...@@ -234,7 +234,7 @@ const Kb = () => {
e.preventDefault(); e.preventDefault();
const targetId = e.currentTarget.getAttribute('data-drag-id'); const targetId = e.currentTarget.getAttribute('data-drag-id');
if (!targetId) return; if (!targetId) return;
KbTypeEnum.folder && setDragTargetId(targetId); DatasetTypeEnum.folder && setDragTargetId(targetId);
}} }}
onDragLeave={(e) => { onDragLeave={(e) => {
e.preventDefault(); e.preventDefault();
...@@ -261,14 +261,14 @@ const Kb = () => { ...@@ -261,14 +261,14 @@ const Kb = () => {
} }
}} }}
onClick={() => { onClick={() => {
if (kb.type === KbTypeEnum.folder) { if (kb.type === DatasetTypeEnum.folder) {
router.push({ router.push({
pathname: '/kb/list', pathname: '/kb/list',
query: { query: {
parentId: kb._id parentId: kb._id
} }
}); });
} else if (kb.type === KbTypeEnum.dataset) { } else if (kb.type === DatasetTypeEnum.dataset) {
router.push({ router.push({
pathname: '/kb/detail', pathname: '/kb/detail',
query: { query: {
...@@ -377,7 +377,7 @@ const Kb = () => { ...@@ -377,7 +377,7 @@ const Kb = () => {
</Flex> </Flex>
</Box> </Box>
<Flex justifyContent={'flex-end'} alignItems={'center'} fontSize={'sm'}> <Flex justifyContent={'flex-end'} alignItems={'center'} fontSize={'sm'}>
{kb.type === KbTypeEnum.folder ? ( {kb.type === DatasetTypeEnum.folder ? (
<Box color={'myGray.500'}>{t('Folder')}</Box> <Box color={'myGray.500'}>{t('Folder')}</Box>
) : ( ) : (
<> <>
......
projects/app/src/pages/login/components/RegisterForm.tsx
...@@ -69,7 +69,8 @@ const RegisterForm = ({ setPageType, loginSuccess }: Props) => { ...@@ -69,7 +69,8 @@ const RegisterForm = ({ setPageType, loginSuccess }: Props) => {
postCreateApp({ postCreateApp({
avatar: template.avatar, avatar: template.avatar,
name: template.name, name: template.name,
modules: template.modules modules: template.modules,
type: template.type
}); });
}); });
}, 100); }, 100);
......
projects/app/src/service/common/bill/push.ts
import { Bill, User, OutLink } from '@/service/mongo'; import { Bill } from '@/service/mongo';
import { MongoUser } from '@fastgpt/support/user/schema';
import { BillSourceEnum } from '@/constants/user'; import { BillSourceEnum } from '@/constants/user';
import { getModel } from '@/service/utils/data'; import { getModel } from '@/service/utils/data';
import { ChatHistoryItemResType } from '@/types/chat'; import { ChatHistoryItemResType } from '@/types/chat';
...@@ -10,7 +11,7 @@ import { defaultQGModel } from '@/pages/api/system/getInitData'; ...@@ -10,7 +11,7 @@ import { defaultQGModel } from '@/pages/api/system/getInitData';
async function createBill(data: CreateBillType) { async function createBill(data: CreateBillType) {
try { try {
await Promise.all([ await Promise.all([
User.findByIdAndUpdate(data.userId, { MongoUser.findByIdAndUpdate(data.userId, {
$inc: { balance: -data.total } $inc: { balance: -data.total }
}), }),
Bill.create(data) Bill.create(data)
...@@ -50,7 +51,7 @@ async function concatBill({ ...@@ -50,7 +51,7 @@ async function concatBill({
} }
} }
), ),
User.findByIdAndUpdate(userId, { MongoUser.findByIdAndUpdate(userId, {
$inc: { balance: -total } $inc: { balance: -total }
}) })
]); ]);
...@@ -177,12 +178,12 @@ export const pushQuestionGuideBill = ({ tokens, userId }: { tokens: number; user ...@@ -177,12 +178,12 @@ export const pushQuestionGuideBill = ({ tokens, userId }: { tokens: number; user
const total = qgModel.price * tokens; const total = qgModel.price * tokens;
createBill({ createBill({
userId, userId,
appName: '问题指引', appName: '下一步指引',
total, total,
source: BillSourceEnum.fastgpt, source: BillSourceEnum.fastgpt,
list: [ list: [
{ {
moduleName: '问题指引', moduleName: '下一步指引',
amount: total, amount: total,
model: qgModel.name, model: qgModel.name,
tokenLen: tokens tokenLen: tokens
......
projects/app/src/service/common/bill/schema.ts
import { Schema, model, models, Model } from 'mongoose'; import { connectionMongo, type Model } from '@fastgpt/common/mongo';
const { Schema, model, models } = connectionMongo;
import { BillSchema as BillType } from '@/types/common/bill'; import { BillSchema as BillType } from '@/types/common/bill';
import { BillSourceMap } from '@/constants/user'; import { BillSourceMap } from '@/constants/user';
......
projects/app/src/service/common/ipLimit/schema.ts deleted 100644 → 0
import { Schema, model, models, Model } from 'mongoose';
import type { IpLimitSchemaType } from '@/types/common/ipLimit';
const IpLimitSchema = new Schema({
eventId: {
type: String,
required: true
},
ip: {
type: String,
required: true
},
account: {
type: Number,
default: 0
},
lastMinute: {
type: Date,
default: () => new Date()
}
});
export const IpLimit: Model<IpLimitSchemaType> =
models['ip_limit'] || model('ip_limit', IpLimitSchema);
projects/app/src/service/dataset/auth.ts
import { isSpecialFileId } from '@fastgpt/core/dataset/utils'; import { isSpecialFileId } from '@fastgpt/core/dataset/utils';
import { GridFSStorage } from '../lib/gridfs'; import { GridFSStorage } from '../lib/gridfs';
import { Types } from 'mongoose'; import { Types } from '@fastgpt/common/mongo';
export async function authFileIdValid(fileId?: string) { export async function authFileIdValid(fileId?: string) {
if (!fileId) return true; if (!fileId) return true;
......
projects/app/src/service/events/generateQA.ts
import { TrainingData } from '@/service/mongo'; import { TrainingData } from '@/service/mongo';
import { pushQABill } from '@/service/common/bill/push'; import { pushQABill } from '@/service/common/bill/push';
import { TrainingModeEnum } from '@/constants/plugin'; import { TrainingModeEnum } from '@/constants/plugin';
import { ERROR_ENUM } from '../errorCode'; import { ERROR_ENUM } from '@fastgpt/common/constant/errorCode';
import { sendInform } from '@/pages/api/user/inform/send'; import { sendInform } from '@/pages/api/user/inform/send';
import { authBalanceByUid } from '../utils/auth'; import { authBalanceByUid } from '@fastgpt/support/user/auth';
import { getAIApi } from '@fastgpt/core/ai/config'; import { getAIApi } from '@fastgpt/core/ai/config';
import type { ChatCompletionRequestMessage } from '@fastgpt/core/ai/type'; import type { ChatCompletionRequestMessage } from '@fastgpt/core/ai/type';
import { addLog } from '../utils/tools'; import { addLog } from '../utils/tools';
......
projects/app/src/service/events/generateVector.ts
import { insertData2Dataset } from '@/service/pg'; import { insertData2Dataset } from '@/service/pg';
import { getVector } from '@/pages/api/openapi/plugin/vector'; import { getVector } from '@/pages/api/openapi/plugin/vector';
import { TrainingData } from '../models/trainingData'; import { TrainingData } from '../models/trainingData';
import { ERROR_ENUM } from '../errorCode'; import { ERROR_ENUM } from '@fastgpt/common/constant/errorCode';
import { TrainingModeEnum } from '@/constants/plugin'; import { TrainingModeEnum } from '@/constants/plugin';
import { sendInform } from '@/pages/api/user/inform/send'; import { sendInform } from '@/pages/api/user/inform/send';
import { addLog } from '../utils/tools'; import { addLog } from '../utils/tools';
......
projects/app/src/service/lib/gridfs.ts
import mongoose, { Types } from 'mongoose'; import { Types, connectionMongo } from '@fastgpt/common/mongo';
import fs from 'fs'; import fs from 'fs';
import fsp from 'fs/promises'; import fsp from 'fs/promises';
import { ERROR_ENUM } from '../errorCode'; import { ERROR_ENUM } from '@fastgpt/common/constant/errorCode';
import type { GSFileInfoType } from '@/types/common/file'; import type { GSFileInfoType } from '@/types/common/file';
enum BucketNameEnum { enum BucketNameEnum {
...@@ -18,10 +18,10 @@ export class GridFSStorage { ...@@ -18,10 +18,10 @@ export class GridFSStorage {
this.uid = String(uid); this.uid = String(uid);
} }
Collection() { Collection() {
return mongoose.connection.db.collection(`${this.bucket}.files`); return connectionMongo.connection.db.collection(`${this.bucket}.files`);
} }
GridFSBucket() { GridFSBucket() {
return new mongoose.mongo.GridFSBucket(mongoose.connection.db, { return new connectionMongo.mongo.GridFSBucket(connectionMongo.connection.db, {
bucketName: this.bucket bucketName: this.bucket
}); });
} }
......
projects/app/src/service/models/app.ts
import { Schema, model, models, Model } from 'mongoose'; import { connectionMongo, type Model } from '@fastgpt/common/mongo';
const { Schema, model, models } = connectionMongo;
import { AppSchema as AppType } from '@/types/mongoSchema'; import { AppSchema as AppType } from '@/types/mongoSchema';
const AppSchema = new Schema({ const AppSchema = new Schema({
......
projects/app/src/service/models/chat.ts
import { Schema, model, models, Model } from 'mongoose'; import { connectionMongo, type Model } from '@fastgpt/common/mongo';
const { Schema, model, models } = connectionMongo;
import { ChatSchema as ChatType } from '@/types/mongoSchema'; import { ChatSchema as ChatType } from '@/types/mongoSchema';
import { ChatRoleMap, TaskResponseKeyEnum } from '@/constants/chat'; import { ChatRoleMap, TaskResponseKeyEnum } from '@/constants/chat';
import { ChatSourceMap } from '@/constants/chat'; import { ChatSourceMap } from '@/constants/chat';
......
projects/app/src/service/models/chatItem.ts
import { Schema, model, models, Model } from 'mongoose'; import { connectionMongo, type Model } from '@fastgpt/common/mongo';
const { Schema, model, models } = connectionMongo;
import { ChatItemSchema as ChatItemType } from '@/types/mongoSchema'; import { ChatItemSchema as ChatItemType } from '@/types/mongoSchema';
import { ChatRoleMap, TaskResponseKeyEnum } from '@/constants/chat'; import { ChatRoleMap, TaskResponseKeyEnum } from '@/constants/chat';
import { customAlphabet } from 'nanoid'; import { customAlphabet } from 'nanoid';
......
projects/app/src/service/models/collection.ts
import { Schema, model, models, Model as MongoModel } from 'mongoose'; import { connectionMongo, type Model } from '@fastgpt/common/mongo';
const { Schema, model, models } = connectionMongo;
import { CollectionSchema as CollectionType } from '@/types/mongoSchema'; import { CollectionSchema as CollectionType } from '@/types/mongoSchema';
const CollectionSchema = new Schema({ const CollectionSchema = new Schema({
...@@ -14,5 +15,5 @@ const CollectionSchema = new Schema({ ...@@ -14,5 +15,5 @@ const CollectionSchema = new Schema({
} }
}); });
export const Collection: MongoModel<CollectionType> = export const Collection: Model<CollectionType> =
models['collection'] || model('collection', CollectionSchema); models['collection'] || model('collection', CollectionSchema);
projects/app/src/service/models/image.ts
import { Schema, model, models, Model } from 'mongoose'; import { connectionMongo, type Model } from '@fastgpt/common/mongo';
const { Schema, model, models } = connectionMongo;
const ImageSchema = new Schema({ const ImageSchema = new Schema({
userId: { userId: {
......
projects/app/src/service/models/inform.ts
import { Schema, model, models, Model } from 'mongoose'; import { connectionMongo, type Model } from '@fastgpt/common/mongo';
const { Schema, model, models } = connectionMongo;
import { informSchema } from '@/types/mongoSchema'; import { informSchema } from '@/types/mongoSchema';
import { InformTypeMap } from '@/constants/user'; import { InformTypeMap } from '@/constants/user';
......
projects/app/src/service/models/pay.ts
import { Schema, model, models, Model } from 'mongoose'; import { connectionMongo, type Model } from '@fastgpt/common/mongo';
const { Schema, model, models } = connectionMongo;
import { PaySchema as PayType } from '@/types/mongoSchema'; import { PaySchema as PayType } from '@/types/mongoSchema';
const PaySchema = new Schema({ const PaySchema = new Schema({
userId: { userId: {
......
projects/app/src/service/models/promotionRecord.ts
import { Schema, model, models, Model } from 'mongoose'; import { connectionMongo, type Model } from '@fastgpt/common/mongo';
const { Schema, model, models } = connectionMongo;
import { PromotionRecordSchema as PromotionRecordType } from '@/types/mongoSchema'; import { PromotionRecordSchema as PromotionRecordType } from '@/types/mongoSchema';
const PromotionRecordSchema = new Schema({ const PromotionRecordSchema = new Schema({
......
projects/app/src/service/models/trainingData.ts
/* 模型的知识库 */ /* 模型的知识库 */
import { Schema, model, models, Model as MongoModel } from 'mongoose'; import { connectionMongo, type Model } from '@fastgpt/common/mongo';
const { Schema, model, models } = connectionMongo;
import { TrainingDataSchema as TrainingDateType } from '@/types/mongoSchema'; import { TrainingDataSchema as TrainingDateType } from '@/types/mongoSchema';
import { TrainingTypeMap } from '@/constants/plugin'; import { TrainingTypeMap } from '@/constants/plugin';
...@@ -68,5 +69,5 @@ try { ...@@ -68,5 +69,5 @@ try {
console.log(error); console.log(error);
} }
export const TrainingData: MongoModel<TrainingDateType> = export const TrainingData: Model<TrainingDateType> =
models['trainingData'] || model('trainingData', TrainingDataSchema); models['trainingData'] || model('trainingData', TrainingDataSchema);
projects/app/src/service/moduleDispatch/agent/classifyQuestion.ts
...@@ -105,7 +105,7 @@ async function functionCall({ ...@@ -105,7 +105,7 @@ async function functionCall({
required: ['type'] required: ['type']
} }
}; };
const ai = getAIApi(user.openaiAccount); const ai = getAIApi(user.openaiAccount, 48000);
const response = await ai.chat.completions.create({ const response = await ai.chat.completions.create({
model: cqModel.model, model: cqModel.model,
......
projects/app/src/service/moduleDispatch/agent/extract.ts
...@@ -126,7 +126,7 @@ async function functionCall({ ...@@ -126,7 +126,7 @@ async function functionCall({
} }
}; };
const ai = getAIApi(user.openaiAccount); const ai = getAIApi(user.openaiAccount, 480000);
const response = await ai.chat.completions.create({ const response = await ai.chat.completions.create({
model: extractModel.model, model: extractModel.model,
......
projects/app/src/service/moduleDispatch/chat/oneapi.ts
...@@ -10,7 +10,7 @@ import { TaskResponseKeyEnum } from '@/constants/chat'; ...@@ -10,7 +10,7 @@ import { TaskResponseKeyEnum } from '@/constants/chat';
import { getChatModel } from '@/service/utils/data'; import { getChatModel } from '@/service/utils/data';
import { countModelPrice } from '@/service/common/bill/push'; import { countModelPrice } from '@/service/common/bill/push';
import { ChatModelItemType } from '@/types/model'; import { ChatModelItemType } from '@/types/model';
import { postTextCensor } from '@/service/common/api/plugins'; import { postTextCensor } from '@fastgpt/common/plusApi/censor';
import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/core/ai/constant'; import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/core/ai/constant';
import { AppModuleItemType } from '@/types/app'; import { AppModuleItemType } from '@/types/app';
import { countMessagesTokens, sliceMessagesTB } from '@/utils/common/tiktoken'; import { countMessagesTokens, sliceMessagesTB } from '@/utils/common/tiktoken';
...@@ -151,7 +151,7 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise<ChatResp ...@@ -151,7 +151,7 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise<ChatResp
}; };
} else { } else {
const unStreamResponse = response as ChatCompletion; const unStreamResponse = response as ChatCompletion;
const answer = unStreamResponse.choices?.[0].message?.content || ''; const answer = unStreamResponse.choices?.[0]?.message?.content || '';
const totalTokens = unStreamResponse.usage?.total_tokens || 0; const totalTokens = unStreamResponse.usage?.total_tokens || 0;
const completeMessages = filterMessages.concat({ const completeMessages = filterMessages.concat({
......
projects/app/src/service/moduleDispatch/kb/search.ts
...@@ -24,6 +24,7 @@ export type KBSearchResponse = { ...@@ -24,6 +24,7 @@ export type KBSearchResponse = {
export async function dispatchKBSearch(props: Record<string, any>): Promise<KBSearchResponse> { export async function dispatchKBSearch(props: Record<string, any>): Promise<KBSearchResponse> {
const { const {
moduleName, moduleName,
user,
inputs: { kbList = [], similarity = 0.4, limit = 5, userChatInput } inputs: { kbList = [], similarity = 0.4, limit = 5, userChatInput }
} = props as KBSearchProps; } = props as KBSearchProps;
...@@ -45,10 +46,10 @@ export async function dispatchKBSearch(props: Record<string, any>): Promise<KBSe ...@@ -45,10 +46,10 @@ export async function dispatchKBSearch(props: Record<string, any>): Promise<KBSe
// search kb // search kb
const res: any = await PgClient.query( const res: any = await PgClient.query(
`BEGIN; `BEGIN;
SET LOCAL ivfflat.probes = ${global.systemEnv.pgIvfflatProbe || 10}; SET LOCAL hnsw.ef_search = ${global.systemEnv.pgHNSWEfSearch || 40};
select id, kb_id, q, a, source, file_id, (vector <#> '[${ select id, kb_id, q, a, source, file_id, (vector <#> '[${
vectors[0] vectors[0]
}]') * -1 AS score from ${PgDatasetTableName} where kb_id IN (${kbList }]') * -1 AS score from ${PgDatasetTableName} where user_id='${user._id}' AND kb_id IN (${kbList
.map((item) => `'${item.kbId}'`) .map((item) => `'${item.kbId}'`)
.join(',')}) AND vector <#> '[${vectors[0]}]' < -${similarity} order by vector <#> '[${ .join(',')}) AND vector <#> '[${vectors[0]}]' < -${similarity} order by vector <#> '[${
vectors[0] vectors[0]
......
projects/app/src/service/mongo.ts
import mongoose from 'mongoose';
import { startQueue } from './utils/tools'; import { startQueue } from './utils/tools';
import { getInitConfig } from '@/pages/api/system/getInitData';
import { User } from './models/user';
import { PRICE_SCALE } from '@fastgpt/common/bill/constants'; import { PRICE_SCALE } from '@fastgpt/common/bill/constants';
import { initPg } from './pg'; import { initPg } from './pg';
import { createHashPassword } from '@/utils/tools'; import { MongoUser } from '@fastgpt/support/user/schema';
import { createLogger, format, transports } from 'winston'; import { connectMongo } from '@fastgpt/common/mongo/init';
import 'winston-mongodb'; import { hashStr } from '@fastgpt/common/tools/str';
import { getTikTokenEnc } from '@/utils/common/tiktoken'; import { getInitConfig, initGlobal } from '@/pages/api/system/getInitData';
import { initHttpAgent } from '@fastgpt/core/init';
/** /**
* connect MongoDB and init data * connect MongoDB and init data
*/ */
export async function connectToDatabase(): Promise<void> { export async function connectToDatabase(): Promise<void> {
if (global.mongodb) { await connectMongo({
return; beforeHook: () => {
} initGlobal();
global.mongodb = 'connecting';
// init global data
global.qaQueueLen = 0;
global.vectorQueueLen = 0;
global.sendInformQueue = [];
global.sendInformQueueLen = 0;
// logger
initLogger();
// init function
getInitConfig(); getInitConfig();
// init tikToken },
getTikTokenEnc(); afterHook: async () => {
initHttpAgent();
try {
mongoose.set('strictQuery', true);
global.mongodb = await mongoose.connect(process.env.MONGODB_URI as string, {
bufferCommands: true,
maxConnecting: Number(process.env.DB_MAX_LINK || 5),
maxPoolSize: Number(process.env.DB_MAX_LINK || 5),
minPoolSize: 2
});
await initRootUser(); await initRootUser();
initPg(); initPg();
console.log('mongo connected'); // start queue
} catch (error) {
console.log('error->', 'mongo connect error');
global.mongodb = null;
}
// init function
startQueue(); startQueue();
}
function initLogger() {
global.logger = createLogger({
transports: [
new transports.MongoDB({
db: process.env.MONGODB_URI as string,
collection: 'server_logs',
options: {
useUnifiedTopology: true
},
cappedSize: 500000000,
tryReconnect: true,
metaKey: 'meta',
format: format.combine(format.timestamp(), format.json())
}),
new transports.Console({
format: format.combine(
format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
format.printf((info) => {
if (info.level === 'error') {
console.log(info.meta);
return `[${info.level.toLocaleUpperCase()}]: ${[info.timestamp]}: ${info.message}`;
} }
return `[${info.level.toLocaleUpperCase()}]: ${[info.timestamp]}: ${info.message}${
info.meta ? `: ${JSON.stringify(info.meta)}` : ''
}`;
})
)
})
]
}); });
} }
async function initRootUser() { async function initRootUser() {
try { try {
const rootUser = await User.findOne({ const rootUser = await MongoUser.findOne({
username: 'root' username: 'root'
}); });
const psw = process.env.DEFAULT_ROOT_PSW || '123456'; const psw = process.env.DEFAULT_ROOT_PSW || '123456';
if (rootUser) { if (rootUser) {
await User.findOneAndUpdate( await MongoUser.findOneAndUpdate(
{ username: 'root' }, { username: 'root' },
{ {
password: createHashPassword(psw), password: hashStr(psw),
balance: 999999 * PRICE_SCALE balance: 999999 * PRICE_SCALE
} }
); );
} else { } else {
await User.create({ await MongoUser.create({
username: 'root', username: 'root',
password: createHashPassword(psw), password: hashStr(psw),
balance: 999999 * PRICE_SCALE balance: 999999 * PRICE_SCALE
}); });
} }
...@@ -121,15 +59,10 @@ async function initRootUser() { ...@@ -121,15 +59,10 @@ async function initRootUser() {
export * from './models/chat'; export * from './models/chat';
export * from './models/chatItem'; export * from './models/chatItem';
export * from './models/app'; export * from './models/app';
export * from './models/user';
export * from './common/bill/schema'; export * from './common/bill/schema';
export * from './models/pay'; export * from './models/pay';
export * from './models/trainingData'; export * from './models/trainingData';
export * from './models/promotionRecord'; export * from './models/promotionRecord';
export * from './models/collection'; export * from './models/collection';
export * from './models/kb';
export * from './models/inform'; export * from './models/inform';
export * from './models/image'; export * from './models/image';
export * from './support/outLink/schema';
export * from './support/openapi/schema';
projects/app/src/service/pg.ts
...@@ -228,6 +228,7 @@ export async function initPg() { ...@@ -228,6 +228,7 @@ export async function initPg() {
q TEXT NOT NULL, q TEXT NOT NULL,
a TEXT a TEXT
); );
CREATE INDEX IF NOT EXISTS vector_index ON ${PgDatasetTableName} USING hnsw (vector vector_ip_ops) WITH (m = 16, ef_construction = 64);
`); `);
console.log('init pg successful'); console.log('init pg successful');
} catch (error) { } catch (error) {
......
projects/app/src/service/response.ts
import { sseResponseEventEnum } from '@/constants/chat'; import { sseResponseEventEnum } from '@/constants/chat';
import { NextApiResponse } from 'next'; import { NextApiResponse } from 'next';
import { proxyError, ERROR_RESPONSE, ERROR_ENUM } from './errorCode'; import { proxyError, ERROR_RESPONSE, ERROR_ENUM } from '@fastgpt/common/constant/errorCode';
import { clearCookie, sseResponse, addLog } from './utils/tools'; import { clearCookie, sseResponse, addLog } from './utils/tools';
export interface ResponseType<T = any> { export interface ResponseType<T = any> {
......
projects/app/src/service/support/outLink/auth.ts deleted 100644 → 0
import { PRICE_SCALE } from '@fastgpt/common/bill/constants';
import { IpLimit } from '@/service/common/ipLimit/schema';
import { authBalanceByUid, AuthUserTypeEnum } from '@/service/utils/auth';
import { OutLinkSchema } from '@/types/support/outLink';
import { OutLink } from './schema';
import axios from 'axios';
type AuthLinkProps = { ip?: string | null; authToken?: string; question: string };
export async function authOutLinkChat({
shareId,
ip,
authToken,
question
}: AuthLinkProps & {
shareId: string;
}) {
// get outLink
const outLink = await OutLink.findOne({
shareId
});
if (!outLink) {
return Promise.reject('分享链接无效');
}
const uid = String(outLink.userId);
const [user] = await Promise.all([
authBalanceByUid(uid), // authBalance
...(global.feConfigs?.isPlus ? [authOutLinkLimit({ outLink, ip, authToken, question })] : []) // limit auth
]);
return {
user,
userId: String(outLink.userId),
appId: String(outLink.appId),
authType: AuthUserTypeEnum.token,
responseDetail: outLink.responseDetail
};
}
export async function authOutLinkLimit({
outLink,
ip,
authToken,
question
}: AuthLinkProps & {
outLink: OutLinkSchema;
}) {
if (!ip || !outLink.limit) {
return;
}
if (outLink.limit.expiredTime && outLink.limit.expiredTime.getTime() < Date.now()) {
return Promise.reject('分享链接已过期');
}
if (outLink.limit.credit > -1 && outLink.total > outLink.limit.credit * PRICE_SCALE) {
return Promise.reject('链接超出使用限制');
}
// ip limit
await (async () => {
if (!outLink.limit) {
return;
}
try {
const ipLimit = await IpLimit.findOne({ ip, eventId: outLink._id });
// first request
if (!ipLimit) {
return await IpLimit.create({
eventId: outLink._id,
ip,
account: outLink.limit.QPM - 1
});
}
// over one minute
const diffTime = Date.now() - ipLimit.lastMinute.getTime();
if (diffTime >= 60 * 1000) {
ipLimit.account = outLink.limit.QPM - 1;
ipLimit.lastMinute = new Date();
return await ipLimit.save();
}
// over limit
if (ipLimit.account <= 0) {
return Promise.reject(
`每分钟仅能请求 ${outLink.limit.QPM} 次, ${60 - Math.round(diffTime / 1000)}s 后重试~`
);
}
// update limit
ipLimit.account = ipLimit.account - 1;
await ipLimit.save();
} catch (error) {}
})();
// url auth. send request
await authShareStart({ authToken, tokenUrl: outLink.limit.hookUrl, question });
}
export async function authOutLinkId({ id }: { id: string }) {
const outLink = await OutLink.findOne({
shareId: id
});
if (!outLink) {
return Promise.reject('分享链接无效');
}
return {
userId: String(outLink.userId)
};
}
type TokenAuthResponseType = {
success: boolean;
msg?: string;
message?: string;
};
export const authShareChatInit = async (authToken?: string, tokenUrl?: string) => {
if (!tokenUrl || !global.feConfigs?.isPlus) return;
try {
const { data } = await axios<TokenAuthResponseType>({
baseURL: tokenUrl,
url: '/shareAuth/init',
method: 'POST',
data: {
token: authToken
}
});
if (data?.success !== true) {
return Promise.reject(data?.message || data?.msg || '身份校验失败');
}
} catch (error) {
return Promise.reject('身份校验失败');
}
};
export const authShareStart = async ({
tokenUrl,
authToken,
question
}: {
authToken?: string;
question: string;
tokenUrl?: string;
}) => {
if (!tokenUrl || !global.feConfigs?.isPlus) return;
try {
const { data } = await axios<TokenAuthResponseType>({
baseURL: tokenUrl,
url: '/shareAuth/start',
method: 'POST',
data: {
token: authToken,
question
}
});
if (data?.success !== true) {
return Promise.reject(data?.message || data?.msg || '身份校验失败');
}
} catch (error) {
return Promise.reject('身份校验失败');
}
};
projects/app/src/service/utils/auth.ts
import type { NextApiRequest } from 'next'; import { App } from '../mongo';
import Cookie from 'cookie'; import { MongoDataset } from '@fastgpt/core/dataset/schema';
import { App, User, KB } from '../mongo'; import type { AppSchema } from '@/types/mongoSchema';
import type { AppSchema, UserModelSchema } from '@/types/mongoSchema'; import { ERROR_ENUM } from '@fastgpt/common/constant/errorCode';
import { ERROR_ENUM } from '../errorCode';
import { authJWT } from './tools';
import { authOpenApiKey } from '../support/openapi/auth';
import { authOutLinkId } from '../support/outLink/auth';
export enum AuthUserTypeEnum {
token = 'token',
root = 'root',
apikey = 'apikey',
outLink = 'outLink'
}
/* auth balance */
export const authBalanceByUid = async (uid: string) => {
const user = await User.findById<UserModelSchema>(
uid,
'_id username balance openaiAccount timezone'
);
if (!user) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
if (user.balance <= 0) {
return Promise.reject(ERROR_ENUM.insufficientQuota);
}
return user;
};
/* uniform auth user */
export const authUser = async ({
req,
authToken = false,
authRoot = false,
authApiKey = false,
authBalance = false,
authOutLink
}: {
req: NextApiRequest;
authToken?: boolean;
authRoot?: boolean;
authApiKey?: boolean;
authBalance?: boolean;
authOutLink?: boolean;
}) => {
const authCookieToken = async (cookie?: string, token?: string): Promise<string> => {
// 获取 cookie
const cookies = Cookie.parse(cookie || '');
const cookieToken = cookies.token || token;
if (!cookieToken) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
return await authJWT(cookieToken);
};
// from authorization get apikey
const parseAuthorization = async (authorization?: string) => {
if (!authorization) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
// Bearer fastgpt-xxxx-appId
const auth = authorization.split(' ')[1];
if (!auth) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
const { apikey, appId: authorizationAppid = '' } = await (async () => {
const arr = auth.split('-');
// abandon
if (arr.length === 3) {
return {
apikey: `${arr[0]}-${arr[1]}`,
appId: arr[2]
};
}
if (arr.length === 2) {
return {
apikey: auth
};
}
return Promise.reject(ERROR_ENUM.unAuthorization);
})();
// auth apikey
const { userId, appId: apiKeyAppId = '' } = await authOpenApiKey({ apikey });
return {
uid: userId,
apikey,
appId: apiKeyAppId || authorizationAppid
};
};
// root user
const parseRootKey = async (rootKey?: string, userId = '') => {
if (!rootKey || !process.env.ROOT_KEY || rootKey !== process.env.ROOT_KEY) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
return userId;
};
const { cookie, token, apikey, rootkey, userid, authorization } = (req.headers || {}) as {
cookie?: string;
token?: string;
apikey?: string;
rootkey?: string; // abandon
userid?: string;
authorization?: string;
};
const { shareId } = (req?.body || {}) as { shareId?: string };
let uid = '';
let appId = '';
let openApiKey = apikey;
let authType: `${AuthUserTypeEnum}` = AuthUserTypeEnum.token;
if (authOutLink && shareId) {
const res = await authOutLinkId({ id: shareId });
uid = res.userId;
authType = AuthUserTypeEnum.outLink;
} else if (authToken && (cookie || token)) {
// user token(from fastgpt web)
uid = await authCookieToken(cookie, token);
authType = AuthUserTypeEnum.token;
} else if (authRoot && rootkey) {
// root user
uid = await parseRootKey(rootkey, userid);
authType = AuthUserTypeEnum.root;
} else if (authApiKey && apikey) {
// apikey
const parseResult = await authOpenApiKey({ apikey });
uid = parseResult.userId;
authType = AuthUserTypeEnum.apikey;
openApiKey = parseResult.apikey;
} else if (authApiKey && authorization) {
// apikey from authorization
const authResponse = await parseAuthorization(authorization);
uid = authResponse.uid;
appId = authResponse.appId;
openApiKey = authResponse.apikey;
authType = AuthUserTypeEnum.apikey;
}
// not rootUser and no uid, reject request
if (!rootkey && !uid) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
// balance check
const user = await (() => {
if (authBalance) {
return authBalanceByUid(uid);
}
})();
return {
userId: String(uid),
appId,
authType,
user,
apikey: openApiKey
};
};
// 模型使用权校验 // 模型使用权校验
export const authApp = async ({ export const authApp = async ({
...@@ -200,8 +37,8 @@ export const authApp = async ({ ...@@ -200,8 +37,8 @@ export const authApp = async ({
}; };
// 知识库操作权限 // 知识库操作权限
export const authKb = async ({ kbId, userId }: { kbId: string; userId: string }) => { export const authDataset = async ({ kbId, userId }: { kbId: string; userId: string }) => {
const kb = await KB.findOne({ const kb = await MongoDataset.findOne({
_id: kbId, _id: kbId,
userId userId
}); });
......
projects/app/src/service/utils/tools.ts
import type { NextApiResponse, NextApiHandler, NextApiRequest } from 'next'; import type { NextApiResponse, NextApiHandler, NextApiRequest } from 'next';
import NextCors from 'nextjs-cors'; import NextCors from 'nextjs-cors';
import crypto from 'crypto';
import jwt from 'jsonwebtoken';
import { generateQA } from '../events/generateQA'; import { generateQA } from '../events/generateQA';
import { generateVector } from '../events/generateVector'; import { generateVector } from '../events/generateVector';
import { ERROR_ENUM } from '../errorCode';
/* 密码加密 */
export const hashPassword = (psw: string) => {
return crypto.createHash('sha256').update(psw).digest('hex');
};
/* 生成 token */
export const generateToken = (userId: string) => {
const key = process.env.TOKEN_KEY as string;
const token = jwt.sign(
{
userId,
exp: Math.floor(Date.now() / 1000) + 60 * 60 * 24 * 7
},
key
);
return token;
};
// auth token
export const authJWT = (token: string) =>
new Promise<string>((resolve, reject) => {
const key = process.env.TOKEN_KEY as string;
jwt.verify(token, key, function (err, decoded: any) {
if (err || !decoded?.userId) {
reject(ERROR_ENUM.unAuthorization);
return;
}
resolve(decoded.userId);
});
});
/* set cookie */ /* set cookie */
export const setCookie = (res: NextApiResponse, token: string) => { export const setCookie = (res: NextApiResponse, token: string) => {
res.setHeader( res.setHeader(
...@@ -67,6 +34,7 @@ export function withNextCors(handler: NextApiHandler): NextApiHandler { ...@@ -67,6 +34,7 @@ export function withNextCors(handler: NextApiHandler): NextApiHandler {
/* start task */ /* start task */
export const startQueue = () => { export const startQueue = () => {
if (!global.systemEnv) return;
for (let i = 0; i < global.systemEnv.qaMaxProcess; i++) { for (let i = 0; i < global.systemEnv.qaMaxProcess; i++) {
generateQA(); generateQA();
} }
......
projects/app/src/types/app.d.ts
...@@ -11,7 +11,7 @@ import type { ...@@ -11,7 +11,7 @@ import type {
FlowOutputItemType, FlowOutputItemType,
FlowOutputTargetItemType FlowOutputTargetItemType
} from '@/types/core/app/flow'; } from '@/types/core/app/flow';
import type { AppSchema, ChatSchema, kbSchema } from './mongoSchema'; import type { AppSchema, ChatSchema } from './mongoSchema';
import { ChatModelType } from '@/constants/model'; import { ChatModelType } from '@/constants/model';
import { FlowValueTypeEnum } from '@/constants/flow'; import { FlowValueTypeEnum } from '@/constants/flow';
......
projects/app/src/types/common/ipLimit.d.ts deleted 100644 → 0
export type IpLimitSchemaType = {
_id: string;
eventId: string;
ip: string;
account: number;
lastMinute: Date;
};
projects/app/src/types/core/chat/type.d.ts
import type { ChatCompletionRequestMessage } from '@fastgpt/core/ai/type'; import type { ChatCompletionRequestMessage } from '@fastgpt/core/ai/type';
import type { NextApiResponse } from 'next'; import type { NextApiResponse } from 'next';
import { RunningModuleItemType } from '@/types/app'; import { RunningModuleItemType } from '@/types/app';
import { UserModelSchema } from '@/types/mongoSchema'; import type { UserModelSchema } from '@fastgpt/support/user/type.d';
export type MessageItemType = ChatCompletionRequestMessage & { dataId?: string; content: string }; export type MessageItemType = ChatCompletionRequestMessage & { dataId?: string; content: string };
......
projects/app/src/types/core/dataset/file.d.ts
import { FileStatusEnum } from '@/constants/dataset'; import { FileStatusEnum } from '@fastgpt/core/dataset/constant';
export type DatasetFileItemType = { export type DatasetFileItemType = {
id: string; id: string;
......
projects/app/src/types/core/dataset/index.d.ts
import { FileStatusEnum } from '@/constants/dataset';
import { PgDataItemType } from './data'; import { PgDataItemType } from './data';
import { VectorModelItemType } from '../../model'; import { VectorModelItemType } from '../../model';
import type { kbSchema } from '../../mongoSchema'; import type { DatasetSchemaType } from '@fastgpt/core/dataset/type.d';
export type DatasetsItemType = Omit<kbSchema, 'vectorModel'> & { export type DatasetsItemType = Omit<DatasetSchemaType, 'vectorModel'> & {
vectorModel: VectorModelItemType; vectorModel: VectorModelItemType;
}; };
......
projects/app/src/types/index.d.ts
import type { Mongoose } from 'mongoose';
import type { Pool } from 'pg'; import type { Pool } from 'pg';
import type { Tiktoken } from 'js-tiktoken'; import type { Tiktoken } from 'js-tiktoken';
import type { Logger } from 'winston';
import { import {
ChatModelItemType, ChatModelItemType,
FunctionModelItemType, FunctionModelItemType,
...@@ -19,59 +17,31 @@ export type PagingData<T> = { ...@@ -19,59 +17,31 @@ export type PagingData<T> = {
export type RequestPaging = { pageNum: number; pageSize: number; [key]: any }; export type RequestPaging = { pageNum: number; pageSize: number; [key]: any };
export type FeConfigsType = {
show_emptyChat?: boolean;
show_register?: boolean;
show_appStore?: boolean;
show_contact?: boolean;
show_git?: boolean;
show_doc?: boolean;
show_pay?: boolean;
show_openai_account?: boolean;
show_promotion?: boolean;
hide_app_flow?: boolean;
openAPIUrl?: string;
systemTitle?: string;
authorText?: string;
googleClientVerKey?: string;
isPlus?: boolean;
oauth?: {
github?: string;
google?: string;
};
limit?: {
exportLimitMinutes?: number;
};
scripts?: { [key: string]: string }[];
};
export type SystemEnvType = { export type SystemEnvType = {
pluginBaseUrl?: string; pluginBaseUrl?: string;
openapiPrefix?: string; openapiPrefix?: string;
vectorMaxProcess: number; vectorMaxProcess: number;
qaMaxProcess: number; qaMaxProcess: number;
pgIvfflatProbe: number; pgHNSWEfSearch: number;
}; };
declare global { declare global {
var mongodb: Mongoose | string | null;
var pgClient: Pool | null; var pgClient: Pool | null;
var qaQueueLen: number; var qaQueueLen: number;
var vectorQueueLen: number; var vectorQueueLen: number;
var TikToken: Tiktoken; var TikToken: Tiktoken;
var logger: Logger;
var sendInformQueue: (() => Promise<void>)[]; var sendInformQueue: (() => Promise<void>)[];
var sendInformQueueLen: number; var sendInformQueueLen: number;
var feConfigs: FeConfigsType; var vectorModels: VectorModelItemType[];
var systemEnv: SystemEnvType;
var chatModels: ChatModelItemType[]; var chatModels: ChatModelItemType[];
var qaModel: QAModelItemType; var qaModel: QAModelItemType;
var extractModel: FunctionModelItemType; var extractModel: FunctionModelItemType;
var cqModel: FunctionModelItemType; var cqModel: FunctionModelItemType;
var qgModel: FunctionModelItemType; var qgModel: FunctionModelItemType;
var vectorModels: VectorModelItemType[];
var priceMd: string;
var systemVersion: string; var systemVersion: string;
interface Window { interface Window {
......
projects/app/src/types/mongoSchema.d.ts
...@@ -6,27 +6,6 @@ import { TrainingModeEnum } from '@/constants/plugin'; ...@@ -6,27 +6,6 @@ import { TrainingModeEnum } from '@/constants/plugin';
import type { AppModuleItemType } from './app'; import type { AppModuleItemType } from './app';
import { ChatSourceEnum } from '@/constants/chat'; import { ChatSourceEnum } from '@/constants/chat';
import { AppTypeEnum } from '@/constants/app'; import { AppTypeEnum } from '@/constants/app';
import { KbTypeEnum } from '@/constants/dataset';
export interface UserModelSchema {
_id: string;
username: string;
password: string;
avatar: string;
balance: number;
promotionRate: number;
inviterId?: string;
openaiKey: string;
createTime: number;
timezone: string;
openaiAccount?: {
key: string;
baseUrl: string;
};
limit: {
exportKbTime?: Date;
};
}
export interface AuthCodeSchema { export interface AuthCodeSchema {
_id: string; _id: string;
...@@ -121,18 +100,6 @@ export interface PromotionRecordSchema { ...@@ -121,18 +100,6 @@ export interface PromotionRecordSchema {
amount: number; amount: number;
} }
export type kbSchema = {
_id: string;
userId: string;
parentId: string;
updateTime: Date;
avatar: string;
name: string;
vectorModel: string;
tags: string[];
type: `${KbTypeEnum}`;
};
export interface informSchema { export interface informSchema {
_id: string; _id: string;
userId: string; userId: string;
......
projects/app/src/types/user.d.ts
import { BillSourceEnum } from '@/constants/user'; import { BillSourceEnum } from '@/constants/user';
import type { UserModelSchema } from './mongoSchema'; import type { UserModelSchema } from '@fastgpt/support/user/type.d';
import type { BillSchema } from '@/types/common/bill'; import type { BillSchema } from '@/types/common/bill';
export interface UserType { export interface UserType {
......
projects/app/src/utils/file.ts
...@@ -7,25 +7,63 @@ import { countPromptTokens } from './common/tiktoken'; ...@@ -7,25 +7,63 @@ import { countPromptTokens } from './common/tiktoken';
* overlapLen - The size of the before and after Text * overlapLen - The size of the before and after Text
* maxLen > overlapLen * maxLen > overlapLen
*/ */
export const splitText2Chunks = ({ text, maxLen }: { text: string; maxLen: number }) => { export const splitText2Chunks = ({ text = '', maxLen }: { text: string; maxLen: number }) => {
const overlapLen = Math.floor(maxLen * 0.25); // Overlap length const overlapLen = Math.floor(maxLen * 0.2); // Overlap length
const tempMarker = 'SPLIT_HERE_SPLIT_HERE';
try { const stepReg: Record<number, RegExp> = {
const tempMarker = 'SPLIT_HERE'; 0: /(\n)/g,
text = text.replace(/\n{3,}/g, '\n'); 1: /([。]|\.\s)/g,
text = text.replace(/\s/g, ' '); 2: /([！？]|!\s|\?\s)/g,
text = text.replace('\n\n', ''); 3: /([；]|;\s)/g,
4: /([，]|,\s)/g
};
const splitTextRecursively = ({ text = '', step }: { text: string; step: number }) => {
if (text.length <= maxLen) {
return [text];
}
const reg = stepReg[step];
if (!reg) {
// use slice-maxLen to split text
const chunks: string[] = [];
let chunk = '';
for (let i = 0; i < text.length; i += maxLen - overlapLen) {
chunk = text.slice(i, i + maxLen);
chunks.push(chunk);
}
return chunks;
}
// split text by delimiters
const splitTexts = text const splitTexts = text
.replace(/([。！？；]|\.\s|!\s|\?\s|;\s|\n)/g, `$1${tempMarker}`) .replace(reg, `$1${tempMarker}`)
.split(tempMarker) .split(`${tempMarker}`)
.filter((part) => part); .filter((part) => part);
const chunks: string[] = [];
let chunks: string[] = [];
let preChunk = ''; let preChunk = '';
let chunk = ''; let chunk = '';
for (let i = 0; i < splitTexts.length; i++) { for (let i = 0; i < splitTexts.length; i++) {
const text = splitTexts[i]; let text = splitTexts[i];
// chunk over size
if (text.length > maxLen) {
const innerChunks = splitTextRecursively({ text, step: step + 1 });
if (innerChunks.length === 0) continue;
// If the last chunk is too small, it is merged into the next chunk
if (innerChunks[innerChunks.length - 1].length <= maxLen * 0.5) {
text = innerChunks.pop() || '';
chunks = chunks.concat(innerChunks);
} else {
chunks = chunks.concat(innerChunks);
continue;
}
}
chunk += text; chunk += text;
// size over lapLen, push it to next chunk
if (chunk.length > maxLen - overlapLen) { if (chunk.length > maxLen - overlapLen) {
preChunk += text; preChunk += text;
} }
...@@ -36,9 +74,14 @@ export const splitText2Chunks = ({ text, maxLen }: { text: string; maxLen: numbe ...@@ -36,9 +74,14 @@ export const splitText2Chunks = ({ text, maxLen }: { text: string; maxLen: numbe
} }
} }
if (chunk) { if (chunk && !chunks[chunks.length - 1].endsWith(chunk)) {
chunks.push(chunk); chunks.push(chunk);
} }
return chunks;
};
try {
const chunks = splitTextRecursively({ text, step: 0 });
const tokens = chunks.reduce((sum, chunk) => sum + countPromptTokens(chunk, 'system'), 0); const tokens = chunks.reduce((sum, chunk) => sum + countPromptTokens(chunk, 'system'), 0);
...@@ -50,14 +93,3 @@ export const splitText2Chunks = ({ text, maxLen }: { text: string; maxLen: numbe ...@@ -50,14 +93,3 @@ export const splitText2Chunks = ({ text, maxLen }: { text: string; maxLen: numbe
throw new Error(getErrText(err)); throw new Error(getErrText(err));
} }
}; };
/* simple text, remove chinese space and extra \n */
export const simpleText = (text: string) => {
text = text.replace(/([\u4e00-\u9fa5])\s+([\u4e00-\u9fa5])/g, '$1$2');
text = text.replace(/\n{2,}/g, '\n');
text = text.replace(/\s{2,}/g, ' ');
text = text.replace(/[\x00-\x08]/g, ' ');
return text;
};
projects/app/src/utils/tools.ts
import crypto from 'crypto';
import dayjs from 'dayjs'; import dayjs from 'dayjs';
/** /**
* 密码加密
*/
export const createHashPassword = (text: string) => {
const hash = crypto.createHash('sha256').update(text).digest('hex');
return hash;
};
/**
* 对象转成 query 字符串 * 对象转成 query 字符串
*/ */
export const Obj2Query = (obj: Record<string, string | number>) => { export const Obj2Query = (obj: Record<string, string | number>) => {
......
projects/app/src/web/common/api/bill.ts
...@@ -16,10 +16,10 @@ export const getPayCode = (amount: number) => ...@@ -16,10 +16,10 @@ export const getPayCode = (amount: number) =>
GET<{ GET<{
codeUrl: string; codeUrl: string;
payId: string; payId: string;
}>(`/plusApi/user/pay/getPayCode`, { amount }); }>(`/plusApi/support/user/pay/getPayCode`, { amount });
export const checkPayResult = (payId: string) => export const checkPayResult = (payId: string) =>
GET<number>(`/plusApi/user/pay/checkPayResult`, { payId }).then(() => { GET<number>(`/plusApi/support/user/pay/checkPayResult`, { payId }).then(() => {
try { try {
GET('/user/account/paySuccess'); GET('/user/account/paySuccess');
} catch (error) {} } catch (error) {}
......
projects/app/src/web/common/api/request.ts
...@@ -5,7 +5,7 @@ import axios, { ...@@ -5,7 +5,7 @@ import axios, {
AxiosProgressEvent AxiosProgressEvent
} from 'axios'; } from 'axios';
import { clearToken, getToken } from '@/utils/user'; import { clearToken, getToken } from '@/utils/user';
import { TOKEN_ERROR_CODE } from '@/service/errorCode'; import { TOKEN_ERROR_CODE } from '@fastgpt/common/constant/errorCode';
interface ConfigType { interface ConfigType {
headers?: { [key: string]: string }; headers?: { [key: string]: string };
......
projects/app/src/web/common/api/system.ts
...@@ -12,6 +12,7 @@ export const postUploadFiles = ( ...@@ -12,6 +12,7 @@ export const postUploadFiles = (
onUploadProgress: (progressEvent: AxiosProgressEvent) => void onUploadProgress: (progressEvent: AxiosProgressEvent) => void
) => ) =>
POST<string[]>('/system/file/upload', data, { POST<string[]>('/system/file/upload', data, {
timeout: 60000,
onUploadProgress, onUploadProgress,
headers: { headers: {
'Content-Type': 'multipart/form-data; charset=utf-8' 'Content-Type': 'multipart/form-data; charset=utf-8'
......
projects/app/src/web/common/store/static.ts
...@@ -6,9 +6,8 @@ import { ...@@ -6,9 +6,8 @@ import {
import type { InitDateResponse } from '@/global/common/api/systemRes'; import type { InitDateResponse } from '@/global/common/api/systemRes';
import { getSystemInitData } from '@/web/common/api/system'; import { getSystemInitData } from '@/web/common/api/system';
import { delay } from '@/utils/tools'; import { delay } from '@/utils/tools';
import { FeConfigsType } from '@/types'; import type { FeConfigsType } from '@fastgpt/common/type/index.d';
export let systemVersion = '0.0.0';
export let chatModelList: ChatModelItemType[] = []; export let chatModelList: ChatModelItemType[] = [];
export let qaModel: QAModelItemType = { export let qaModel: QAModelItemType = {
model: 'gpt-3.5-turbo-16k', model: 'gpt-3.5-turbo-16k',
...@@ -18,6 +17,8 @@ export let qaModel: QAModelItemType = { ...@@ -18,6 +17,8 @@ export let qaModel: QAModelItemType = {
}; };
export let vectorModelList: VectorModelItemType[] = []; export let vectorModelList: VectorModelItemType[] = [];
export let feConfigs: FeConfigsType = {}; export let feConfigs: FeConfigsType = {};
export let priceMd = '';
export let systemVersion = '0.0.0';
let retryTimes = 3; let retryTimes = 3;
...@@ -29,6 +30,7 @@ export const clientInitData = async (): Promise<InitDateResponse> => { ...@@ -29,6 +30,7 @@ export const clientInitData = async (): Promise<InitDateResponse> => {
qaModel = res.qaModel; qaModel = res.qaModel;
vectorModelList = res.vectorModels; vectorModelList = res.vectorModels;
feConfigs = res.feConfigs; feConfigs = res.feConfigs;
priceMd = res.priceMd;
systemVersion = res.systemVersion; systemVersion = res.systemVersion;
return res; return res;
......
projects/app/src/web/core/api/dataset.ts
...@@ -12,7 +12,7 @@ import type { ...@@ -12,7 +12,7 @@ import type {
GetDatasetDataListProps GetDatasetDataListProps
} from '@/global/core/api/datasetReq.d'; } from '@/global/core/api/datasetReq.d';
import type { SearchTestResponseType, PushDataResponse } from '@/global/core/api/datasetRes.d'; import type { SearchTestResponseType, PushDataResponse } from '@/global/core/api/datasetRes.d';
import { KbTypeEnum } from '@/constants/dataset'; import { DatasetTypeEnum } from '@fastgpt/core/dataset/constant';
import type { DatasetFileItemType } from '@/types/core/dataset/file'; import type { DatasetFileItemType } from '@/types/core/dataset/file';
import type { GSFileInfoType } from '@/types/common/file'; import type { GSFileInfoType } from '@/types/common/file';
import type { QuoteItemType } from '@/types/chat'; import type { QuoteItemType } from '@/types/chat';
...@@ -21,7 +21,7 @@ import download from 'downloadjs'; ...@@ -21,7 +21,7 @@ import download from 'downloadjs';
import type { DatasetDataItemType } from '@/types/core/dataset/data'; import type { DatasetDataItemType } from '@/types/core/dataset/data';
/* ======================== dataset ======================= */ /* ======================== dataset ======================= */
export const getDatasets = (data: { parentId?: string; type?: `${KbTypeEnum}` }) => export const getDatasets = (data: { parentId?: string; type?: `${DatasetTypeEnum}` }) =>
GET<DatasetsItemType[]>(`/core/dataset/list`, data); GET<DatasetsItemType[]>(`/core/dataset/list`, data);
/** /**
......
projects/app/src/web/support/api/openapi.ts
import { GET, POST, DELETE } from '@/web/common/api/request'; import { GET, POST, DELETE } from '@/web/common/api/request';
import type { EditApiKeyProps, GetApiKeyProps } from '@/global/support/api/openapiReq.d'; import type { EditApiKeyProps, GetApiKeyProps } from '@/global/support/api/openapiReq.d';
import type { OpenApiSchema } from '@/types/support/openapi'; import type { OpenApiSchema } from '@fastgpt/support/openapi/type.d';
/** /**
* crete a api key * crete a api key
......
projects/app/src/web/support/api/outLink.ts
import { GET, POST, DELETE } from '@/web/common/api/request'; import { GET, POST, DELETE } from '@/web/common/api/request';
import type { InitShareChatResponse } from '@/global/support/api/outLinkRes.d'; import type { InitShareChatResponse } from '@/global/support/api/outLinkRes.d';
import type { OutLinkEditType } from '@/types/support/outLink'; import type { OutLinkEditType, OutLinkSchema } from '@fastgpt/support/outLink/type.d';
import type { OutLinkSchema } from '@/types/support/outLink';
/** /**
* 初始化分享聊天 * 初始化分享聊天
......
projects/app/src/web/support/api/user.ts
import { GET, POST, PUT } from '@/web/common/api/request'; import { GET, POST, PUT } from '@/web/common/api/request';
import { createHashPassword } from '@/utils/tools'; import { hashStr } from '@fastgpt/common/tools/str';
import type { ResLogin, PromotionRecordType } from '@/global/support/api/userRes.d'; import type { ResLogin, PromotionRecordType } from '@/global/support/api/userRes.d';
import { UserAuthTypeEnum } from '@/constants/common'; import { UserAuthTypeEnum } from '@/constants/common';
import { UserType, UserUpdateParams } from '@/types/user'; import { UserType, UserUpdateParams } from '@/types/user';
...@@ -11,7 +11,7 @@ export const sendAuthCode = (data: { ...@@ -11,7 +11,7 @@ export const sendAuthCode = (data: {
username: string; username: string;
type: `${UserAuthTypeEnum}`; type: `${UserAuthTypeEnum}`;
googleToken: string; googleToken: string;
}) => POST(`/plusApi/user/inform/sendAuthCode`, data); }) => POST(`/plusApi/support/user/inform/sendAuthCode`, data);
export const getTokenLogin = () => GET<UserType>('/user/account/tokenLogin'); export const getTokenLogin = () => GET<UserType>('/user/account/tokenLogin');
export const oauthLogin = (params: { export const oauthLogin = (params: {
...@@ -19,7 +19,7 @@ export const oauthLogin = (params: { ...@@ -19,7 +19,7 @@ export const oauthLogin = (params: {
code: string; code: string;
callbackUrl: string; callbackUrl: string;
inviterId?: string; inviterId?: string;
}) => POST<ResLogin>('/plusApi/user/account/login/oauth', params); }) => POST<ResLogin>('/plusApi/support/user/account/login/oauth', params);
export const postRegister = ({ export const postRegister = ({
username, username,
...@@ -32,11 +32,11 @@ export const postRegister = ({ ...@@ -32,11 +32,11 @@ export const postRegister = ({
password: string; password: string;
inviterId?: string; inviterId?: string;
}) => }) =>
POST<ResLogin>(`/plusApi/user/account/register/emailAndPhone`, { POST<ResLogin>(`/plusApi/support/user/account/register/emailAndPhone`, {
username, username,
code, code,
inviterId, inviterId,
password: createHashPassword(password) password: hashStr(password)
}); });
export const postFindPassword = ({ export const postFindPassword = ({
...@@ -48,22 +48,22 @@ export const postFindPassword = ({ ...@@ -48,22 +48,22 @@ export const postFindPassword = ({
code: string; code: string;
password: string; password: string;
}) => }) =>
POST<ResLogin>(`/plusApi/user/account/password/updateByCode`, { POST<ResLogin>(`/plusApi/support/user/account/password/updateByCode`, {
username, username,
code, code,
password: createHashPassword(password) password: hashStr(password)
}); });
export const updatePasswordByOld = ({ oldPsw, newPsw }: { oldPsw: string; newPsw: string }) => export const updatePasswordByOld = ({ oldPsw, newPsw }: { oldPsw: string; newPsw: string }) =>
POST('/user/account/updatePasswordByOld', { POST('/user/account/updatePasswordByOld', {
oldPsw: createHashPassword(oldPsw), oldPsw: hashStr(oldPsw),
newPsw: createHashPassword(newPsw) newPsw: hashStr(newPsw)
}); });
export const postLogin = ({ username, password }: { username: string; password: string }) => export const postLogin = ({ username, password }: { username: string; password: string }) =>
POST<ResLogin>('/user/account/loginByPassword', { POST<ResLogin>('/user/account/loginByPassword', {
username, username,
password: createHashPassword(password) password: hashStr(password)
}); });
export const loginOut = () => GET('/user/account/loginout'); export const loginOut = () => GET('/user/account/loginout');
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment