feat: content check

51a5d450 · archer · 98444fd0 · 51a5d450 · 51a5d450 · 51a5d450
Commit 51a5d450 authored May 21, 2023 by archer
15 changed files
--- a/.env.template
+++ b/.env.template
@@ -16,6 +16,8 @@ aliTemplateCode=SMS_xxx
 TOKEN_KEY=xxx
 # root key, 最高权限
 ROOT_KEY=xxx
+# 是否进行安全校验(1: 开启，0: 关闭)
+SENSITIVE_CHECK=1
 # openai
 # OPENAI_BASE_URL=https://api.openai.com/v1
 # OPENAI_BASE_URL_AUTH=可选的安全凭证(不需要的时候，记得去掉)

--- a/Dockerfile
+++ b/Dockerfile
@@ -52,6 +52,8 @@ COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static
 USER nextjs
+ENV PORT=3000
 EXPOSE 3000
 CMD ["node", "server.js"]
--- a/docs/deploy/fastgpt/docker-compose.yml
+++ b/docs/deploy/fastgpt/docker-compose.yml
@@ -52,6 +52,10 @@ services:
      - aliTemplateCode=SMS_xxxx
      # token加密凭证（随便填，作为登录凭证）
      - TOKEN_KEY=xxxx
+      # root key, 最高权限
+      - ROOT_KEY=xxx
+      # 是否进行安全校验(1: 开启，0: 关闭)
+      - SENSITIVE_CHECK=1
      # 和上方mongo镜像的username,password对应
      - MONGODB_URI=mongodb://username:password@0.0.0.0:27017/?authSource=admin
      - MONGODB_NAME=fastgpt

--- a/docs/dev/README.md
+++ b/docs/dev/README.md
@@ -10,34 +10,38 @@
 # proxy（可选）
 AXIOS_PROXY_HOST=127.0.0.1
 AXIOS_PROXY_PORT=7890
-# openai 中转连接（可选）
+# 是否开启队列任务。 1-开启，0-关闭（请求parentUrl去执行任务,单机时直接填1）
-OPENAI_BASE_URL=https://api.openai.com/v1
-OPENAI_BASE_URL_AUTH=可选的安全凭证
-# 是否开启队列任务。 1-开启，0-关闭（请求 parentUrl 去执行任务,单机时直接填1）
 queueTask=1
 parentUrl=https://hostname/api/openapi/startEvents
-# 发送邮箱验证码配置。用的是 QQ 邮箱。参考 nodeMail 获取MAILE_CODE，自行百度。
+# email
-MY_MAIL=xxxx@qq.com
+MY_MAIL=xxx@qq.com
-MAILE_CODE=xxxx
+MAILE_CODE=xxx
-# 阿里短信服务（邮箱和短信至少二选一）
+# ali ems
-aliAccessKeyId=xxxx
+aliAccessKeyId=xxx
-aliAccessKeySecret=xxxx
+aliAccessKeySecret=xxx
-aliSignName=xxxxx
+aliSignName=xxx
-aliTemplateCode=SMS_xxxx
+aliTemplateCode=SMS_xxx
-# token加密凭证（随便填，作为登录凭证）
+# token
-TOKEN_KEY=xxxx
+TOKEN_KEY=xxx
-queueTask=1
+# root key, 最高权限
-parentUrl=https://hostname/api/openapi/startEvents
+ROOT_KEY=xxx
-# 和mongo镜像的username,password对应
+# 是否进行安全校验(1: 开启，0: 关闭)
-MONGODB_URI=mongodb://username:passsword@0.0.0.0:27017/?authSource=admin
+SENSITIVE_CHECK=1
-MONGODB_NAME=xxx
+# openai
+# OPENAI_BASE_URL=https://api.openai.com/v1
+# OPENAI_BASE_URL_AUTH=可选的安全凭证(不需要的时候，记得去掉)
+OPENAIKEY=sk-xxx
+GPT4KEY=sk-xxx
+# claude
+CLAUDE_BASE_URL=calude模型请求地址
+CLAUDE_KEY=CLAUDE_KEY
+# db
+MONGODB_URI=mongodb://username:password@0.0.0.0:27017/test?authSource=admin
 PG_HOST=0.0.0.0
 PG_PORT=8100
-# 和PG镜像对应.
+PG_USER=xxx
-PG_USER=fastgpt # POSTGRES_USER
+PG_PASSWORD=xxx
-PG_PASSWORD=1234 # POSTGRES_PASSWORD
+PG_DB_NAME=xxx
-PG_DB_NAME=fastgpt # POSTGRES_DB
-OPENAIKEY=sk-xxxxx
 ```
 ## 运行

--- a/src/constants/plugin.ts
+++ b/src/constants/plugin.ts
@@ -2,3 +2,13 @@ export enum SplitTextTypEnum {
  'qa' = 'qa',
  'subsection' = 'subsection'
 }
+export enum PluginTypeEnum {
+  LLM = 'LLM',
+  Text = 'Text',
+  Function = 'Function'
+}
+export enum PluginParamsTypeEnum {
+  'Text' = 'text'
+}
--- a/src/pages/api/chat/chat.ts
+++ b/src/pages/api/chat/chat.ts
@@ -10,6 +10,7 @@ import { resStreamResponse } from '@/service/utils/chat';
 import { searchKb } from '@/service/plugins/searchKb';
 import { ChatRoleEnum } from '@/constants/chat';
 import { BillTypeEnum } from '@/constants/user';
+import { sensitiveCheck } from '@/service/api/text';
 /* 发送提示词 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
@@ -44,6 +45,10 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
    // 读取对话内容
    const prompts = [...content, prompt];
+    let systemPrompts: {
+      obj: ChatRoleEnum;
+      value: string;
+    }[] = [];
    // 使用了知识库搜索
    if (model.chat.relatedKbs.length > 0) {
@@ -60,15 +65,22 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
        return res.send(searchPrompts[0]?.value);
      }
-      prompts.splice(prompts.length - 3, 0, ...searchPrompts);
+      systemPrompts = searchPrompts;
-    } else {
+    } else if (model.chat.systemPrompt) {
-      // 没有用知识库搜索，仅用系统提示词
+      systemPrompts = [
-      model.chat.systemPrompt &&
+        {
-        prompts.splice(prompts.length - 3, 0, {
          obj: ChatRoleEnum.System,
          value: model.chat.systemPrompt
-        });
        }
+      ];
+    }
+    prompts.splice(prompts.length - 3, 0, ...systemPrompts);
+    // content check
+    await sensitiveCheck({
+      input: [...systemPrompts, prompt].map((item) => item.value).join('')
+    });
    // 计算温度
    const temperature = (modelConstantsData.maxTemperature * (model.chat.temperature / 10)).toFixed(

--- a/src/pages/api/chat/shareChat/chat.ts
+++ b/src/pages/api/chat/shareChat/chat.ts
@@ -10,6 +10,7 @@ import { resStreamResponse } from '@/service/utils/chat';
 import { searchKb } from '@/service/plugins/searchKb';
 import { ChatRoleEnum } from '@/constants/chat';
 import { BillTypeEnum } from '@/constants/user';
+import { sensitiveCheck } from '@/service/api/text';
 /* 发送提示词 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
@@ -41,6 +42,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
    const modelConstantsData = ChatModelMap[model.chat.chatModel];
+    let systemPrompts: {
+      obj: ChatRoleEnum;
+      value: string;
+    }[] = [];
    // 使用了知识库搜索
    if (model.chat.relatedKbs.length > 0) {
      const { code, searchPrompts } = await searchKb({
@@ -56,15 +62,22 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
        return res.send(searchPrompts[0]?.value);
      }
-      prompts.splice(prompts.length - 3, 0, ...searchPrompts);
+      systemPrompts = searchPrompts;
-    } else {
+    } else if (model.chat.systemPrompt) {
-      // 没有用知识库搜索，仅用系统提示词
+      systemPrompts = [
-      model.chat.systemPrompt &&
+        {
-        prompts.splice(prompts.length - 3, 0, {
          obj: ChatRoleEnum.System,
          value: model.chat.systemPrompt
-        });
        }
+      ];
+    }
+    prompts.splice(prompts.length - 3, 0, ...systemPrompts);
+    // content check
+    await sensitiveCheck({
+      input: [...systemPrompts, prompts[prompts.length - 1]].map((item) => item.value).join('')
+    });
    // 计算温度
    const temperature = (modelConstantsData.maxTemperature * (model.chat.temperature / 10)).toFixed(

--- a/src/pages/api/openapi/chat/chat.ts
+++ b/src/pages/api/openapi/chat/chat.ts
@@ -10,6 +10,7 @@ import { searchKb } from '@/service/plugins/searchKb';
 import { ChatRoleEnum } from '@/constants/chat';
 import { withNextCors } from '@/service/utils/tools';
 import { BillTypeEnum } from '@/constants/user';
+import { sensitiveCheck } from '@/service/api/text';
 /* 发送提示词 */
 export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse) {
@@ -62,13 +63,16 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
    const modelConstantsData = ChatModelMap[model.chat.chatModel];
+    let systemPrompts: {
+      obj: ChatRoleEnum;
+      value: string;
+    }[] = [];
    // 使用了知识库搜索
    if (model.chat.relatedKbs.length > 0) {
-      const similarity = ModelVectorSearchModeMap[model.chat.searchMode]?.similarity || 0.22;
      const { code, searchPrompts } = await searchKb({
        prompts,
-        similarity,
+        similarity: ModelVectorSearchModeMap[model.chat.searchMode]?.similarity,
        model,
        userId
      });
@@ -77,17 +81,28 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
      if (code === 201) {
        return isStream
          ? res.send(searchPrompts[0]?.value)
-          : jsonRes(res, { data: searchPrompts[0]?.value });
+          : jsonRes(res, {
+              data: searchPrompts[0]?.value,
+              message: searchPrompts[0]?.value
+            });
      }
-      prompts.splice(prompts.length - 3, 0, ...searchPrompts);
-    } else {
+      systemPrompts = searchPrompts;
-      // 没有用知识库搜索，仅用系统提示词
+    } else if (model.chat.systemPrompt) {
-      model.chat.systemPrompt &&
+      systemPrompts = [
-        prompts.splice(prompts.length - 3, 0, {
+        {
          obj: ChatRoleEnum.System,
          value: model.chat.systemPrompt
-        });
        }
+      ];
+    }
+    prompts.splice(prompts.length - 3, 0, ...systemPrompts);
+    // content check
+    await sensitiveCheck({
+      input: [...systemPrompts, prompts[prompts.length - 1]].map((item) => item.value).join('')
+    });
    // 计算温度
    const temperature = (modelConstantsData.maxTemperature * (model.chat.temperature / 10)).toFixed(

--- a/src/pages/api/openapi/text/sensitiveCheck.ts
+++ b/src/pages/api/openapi/text/sensitiveCheck.ts
+// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { authUser, getSystemOpenAiKey } from '@/service/utils/auth';
+import type { TextPluginRequestParams } from '@/types/plugin';
+import axios from 'axios';
+import { axiosConfig } from '@/service/utils/tools';
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  try {
+    if (process.env.SENSITIVE_CHECK !== '1') {
+      return jsonRes(res);
+    }
+    await authUser({ req });
+    const { input } = req.body as TextPluginRequestParams;
+    const response = await axios({
+      ...axiosConfig(getSystemOpenAiKey()),
+      method: 'POST',
+      url: `/moderations`,
+      data: {
+        input
+      }
+    });
+    const data = (response.data.results?.[0]?.category_scores as Record<string, number>) || {};
+    const values = Object.values(data);
+    for (const val of values) {
+      if (val > 0.2) {
+        return jsonRes(res, {
+          code: 500,
+          message: '您的内容不合规'
+        });
+      }
+    }
+    jsonRes(res);
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}
--- a/src/service/api/request.ts
+++ b/src/service/api/request.ts
+import axios, { Method, InternalAxiosRequestConfig, AxiosResponse } from 'axios';
+interface ConfigType {
+  headers?: { [key: string]: string };
+  hold?: boolean;
+}
+interface ResponseDataType {
+  code: number;
+  message: string;
+  data: any;
+}
+/**
+ * 请求开始
+ */
+function requestStart(config: InternalAxiosRequestConfig): InternalAxiosRequestConfig {
+  if (config.headers) {
+    config.headers.rootkey = process.env.ROOT_KEY;
+  }
+  return config;
+}
+/**
+ * 请求成功,检查请求头
+ */
+function responseSuccess(response: AxiosResponse<ResponseDataType>) {
+  return response;
+}
+/**
+ * 响应数据检查
+ */
+function checkRes(data: ResponseDataType) {
+  if (data === undefined) {
+    return Promise.reject('服务器异常');
+  } else if (data.code < 200 || data.code >= 400) {
+    return Promise.reject(data);
+  }
+  return data.data;
+}
+/**
+ * 响应错误
+ */
+function responseError(err: any) {
+  if (!err) {
+    return Promise.reject({ message: '未知错误' });
+  }
+  if (typeof err === 'string') {
+    return Promise.reject({ message: err });
+  }
+  return Promise.reject(err);
+}
+/* 创建请求实例 */
+const instance = axios.create({
+  timeout: 60000, // 超时时间
+  headers: {
+    'content-type': 'application/json'
+  }
+});
+/* 请求拦截 */
+instance.interceptors.request.use(requestStart, (err) => Promise.reject(err));
+/* 响应拦截 */
+instance.interceptors.response.use(responseSuccess, (err) => Promise.reject(err));
+function request(url: string, data: any, config: ConfigType, method: Method): any {
+  /* 去空 */
+  for (const key in data) {
+    if (data[key] === null || data[key] === undefined) {
+      delete data[key];
+    }
+  }
+  return instance
+    .request({
+      baseURL: `http://localhost:${process.env.PORT || 3000}/api`,
+      url,
+      method,
+      data: method === 'GET' ? null : data,
+      params: method === 'GET' ? data : null, // get请求不携带data，params放在url上
+      ...config // 用户自定义配置，可以覆盖前面的配置
+    })
+    .then((res) => checkRes(res.data))
+    .catch((err) => responseError(err));
+}
+/**
+ * api请求方式
+ * @param {String} url
+ * @param {Any} params
+ * @param {Object} config
+ * @returns
+ */
+export function GET<T>(url: string, params = {}, config: ConfigType = {}): Promise<T> {
+  return request(url, params, config, 'GET');
+}
+export function POST<T>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
+  return request(url, data, config, 'POST');
+}
+export function PUT<T>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
+  return request(url, data, config, 'PUT');
+}
+export function DELETE<T>(url: string, config: ConfigType = {}): Promise<T> {
+  return request(url, {}, config, 'DELETE');
+}
--- a/src/service/api/text.ts
+++ b/src/service/api/text.ts
+import { POST } from './request';
+import type { TextPluginRequestParams } from '@/types/plugin';
+export const sensitiveCheck = (data: TextPluginRequestParams) =>
+  POST('/openapi/text/sensitiveCheck', data);
--- a/src/service/utils/auth.ts
+++ b/src/service/utils/auth.ts
@@ -66,8 +66,8 @@ export const authUser = async ({
      return Promise.reject(error);
    }
  };
-  const parseRootKey = async (rootKey?: string, userId?: string) => {
+  const parseRootKey = async (rootKey?: string, userId = '') => {
-    if (!rootKey || !userId || !process.env.ROOT_KEY || rootKey !== process.env.ROOT_KEY) {
+    if (!rootKey || !process.env.ROOT_KEY || rootKey !== process.env.ROOT_KEY) {
      return Promise.reject(ERROR_ENUM.unAuthorization);
    }
    return userId;
@@ -104,7 +104,7 @@ export const authUser = async ({
 };
 /* random get openai api key */
-export const getOpenAiKey = () => {
+export const getSystemOpenAiKey = () => {
  // 纯字符串类型
  const keys = process.env.OPENAIKEY?.split(',') || [];
  const i = Math.floor(Math.random() * keys.length);
@@ -129,7 +129,7 @@ export const getApiKey = async ({
  const keyMap = {
    [OpenAiChatEnum.GPT35]: {
      userOpenAiKey: user.openaiKey || '',
-      systemAuthKey: getOpenAiKey() as string
+      systemAuthKey: getSystemOpenAiKey() as string
    },
    [OpenAiChatEnum.GPT4]: {
      userOpenAiKey: user.openaiKey || '',

--- a/src/service/utils/chat/openai.ts
+++ b/src/service/utils/chat/openai.ts
@@ -7,16 +7,14 @@ import { adaptChatItem_openAI } from '@/utils/chat/openai';
 import { modelToolMap } from '@/utils/chat';
 import { ChatCompletionType, ChatContextFilter, StreamResponseType } from './index';
 import { ChatRoleEnum } from '@/constants/chat';
-import { getOpenAiKey } from '../auth';
+import { getSystemOpenAiKey } from '../auth';
-export const getOpenAIApi = (apiKey: string) => {
+export const getOpenAIApi = () =>
-  const configuration = new Configuration({
+  new OpenAIApi(
-    apiKey,
+    new Configuration({
-    basePath: process.env.OPENAI_BASE_URL
+      basePath: process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1'
-  });
+    })
+  );
-  return new OpenAIApi(configuration);
-};
 /* 获取向量 */
 export const openaiCreateEmbedding = async ({
@@ -28,10 +26,10 @@ export const openaiCreateEmbedding = async ({
  userId: string;
  textArr: string[];
 }) => {
-  const systemAuthKey = getOpenAiKey();
+  const systemAuthKey = getSystemOpenAiKey();
  // 获取 chatAPI
-  const chatAPI = getOpenAIApi(userOpenAiKey || systemAuthKey);
+  const chatAPI = getOpenAIApi();
  // 把输入的内容转成向量
  const res = await chatAPI
@@ -42,7 +40,7 @@ export const openaiCreateEmbedding = async ({
      },
      {
        timeout: 60000,
-        ...axiosConfig()
+        ...axiosConfig(userOpenAiKey || systemAuthKey)
      }
    )
    .then((res) => ({
@@ -78,7 +76,7 @@ export const chatResponse = async ({
  });
  const adaptMessages = adaptChatItem_openAI({ messages: filterMessages });
-  const chatAPI = getOpenAIApi(apiKey);
+  const chatAPI = getOpenAIApi();
  const response = await chatAPI.createChatCompletion(
    {
@@ -93,7 +91,7 @@ export const chatResponse = async ({
    {
      timeout: stream ? 60000 : 240000,
      responseType: stream ? 'stream' : 'json',
-      ...axiosConfig()
+      ...axiosConfig(apiKey)
    }
  );

--- a/src/service/utils/tools.ts
+++ b/src/service/utils/tools.ts
@@ -31,9 +31,11 @@ export const clearCookie = (res: NextApiResponse) => {
 };
 /* openai axios config */
-export const axiosConfig = () => ({
+export const axiosConfig = (apikey: string) => ({
+  baseURL: process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1',
  httpsAgent: global.httpsAgent,
  headers: {
+    Authorization: `Bearer ${apikey}`,
    auth: process.env.OPENAI_BASE_URL_AUTH || ''
  }
 });

--- a/src/types/plugin.d.ts
+++ b/src/types/plugin.d.ts
 import type { kbSchema } from './mongoSchema';
+import { PluginTypeEnum } from '@/constants/plugin';
 /* kb type */
 export interface KbItemType extends kbSchema {
  totalData: number;
  tags: string;
 }
 export interface KbDataItemType {
  id: string;
  status: 'waiting' | 'ready';
@@ -13,3 +15,21 @@ export interface KbDataItemType {
  kbId: string;
  userId: string;
 }
+/* plugin */
+export interface PluginConfig {
+  name: string;
+  desc: string;
+  url: string;
+  category: `${PluginTypeEnum}`;
+  uniPrice: 22; // 1k token
+  params: [
+    {
+      type: '';
+    }
+  ];
+}
+export type TextPluginRequestParams = {
+  input: string;
+};