feat: gridfs save file

client/package.json

@@ -37,6 +37,7 @@
     "mammoth": "^1.5.1",
     "mermaid": "^10.2.3",
     "mongoose": "^6.10.0",
+    "multer": "1.4.5-lts.1",
     "nanoid": "^4.0.1",
     "next": "13.1.6",
     "next-i18next": "^13.3.0",
@@ -72,6 +73,7 @@
     "@types/jsdom": "^21.1.1",
     "@types/jsonwebtoken": "^9.0.1",
     "@types/lodash": "^4.14.191",
+    "@types/multer": "^1.4.7",
     "@types/node": "18.14.0",
     "@types/papaparse": "^5.3.7",
     "@types/pg": "^8.6.6",

client/src/api/service/request.ts

 import axios, { Method, InternalAxiosRequestConfig, AxiosResponse } from 'axios';
-import { baseUrl } from '../../service/ai/openai';
+import { baseUrl } from '../../service/lib/openai';
 
 interface ConfigType {
   headers?: { [key: string]: string };

client/src/pages/api/openapi/plugin/vector.ts

@@ -2,7 +2,7 @@ import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { authBalanceByUid, authUser } from '@/service/utils/auth';
 import { withNextCors } from '@/service/utils/tools';
-import { getAIChatApi, axiosConfig } from '@/service/ai/openai';
+import { getAIChatApi, axiosConfig } from '@/service/lib/openai';
 import { pushGenerateVectorBill } from '@/service/events/pushBill';
 
 type Props = {

client/src/pages/api/plugins/file/read.ts

+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase } from '@/service/mongo';
+import { authUser } from '@/service/utils/auth';
+import { GridFSStorage } from '@/service/lib/gridfs';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
+  try {
+    await connectToDatabase();
+
+    const { fileId } = req.query as { fileId: string };
+
+    if (!fileId) {
+      throw new Error('fileId is empty');
+    }
+
+    const { userId } = await authUser({ req });
+
+    const gridFs = new GridFSStorage('dataset', userId);
+
+    const [file, buffer] = await Promise.all([
+      gridFs.findAndAuthFile(fileId),
+      gridFs.download(fileId)
+    ]);
+
+    res.setHeader('encoding', file.encoding);
+    res.setHeader('Content-Type', file.contentType);
+
+    res.end(buffer);
+  } catch (error) {
+    jsonRes(res, {
+      code: 500,
+      error
+    });
+  }
+}

client/src/pages/api/plugins/file/upload.ts

+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase } from '@/service/mongo';
+import { authUser } from '@/service/utils/auth';
+import { GridFSStorage } from '@/service/lib/gridfs';
+import { customAlphabet } from 'nanoid';
+import multer from 'multer';
+import path from 'path';
+
+const nanoid = customAlphabet('1234567890abcdef', 12);
+
+type FileType = {
+  fieldname: string;
+  originalname: string;
+  encoding: string;
+  mimetype: string;
+  filename: string;
+  path: string;
+  size: number;
+};
+
+/**
+ * Creates the multer uploader
+ */
+const maxSize = 50 * 1024 * 1024;
+class UploadModel {
+  uploader = multer({
+    limits: {
+      fieldSize: maxSize
+    },
+    storage: multer.diskStorage({
+      filename: (_req, file, cb) => {
+        const { ext } = path.parse(file.originalname);
+        cb(null, nanoid() + ext);
+      }
+    })
+  }).any();
+
+  async doUpload(req: NextApiRequest, res: NextApiResponse) {
+    return new Promise<{ files: FileType[] }>((resolve, reject) => {
+      // @ts-ignore
+      this.uploader(req, res, (error) => {
+        if (error) {
+          return reject(error);
+        }
+
+        // @ts-ignore
+        resolve({ files: req.files });
+      });
+    });
+  }
+}
+
+const upload = new UploadModel();
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
+  try {
+    await connectToDatabase();
+    const { userId } = await authUser({ req });
+
+    const { files } = await upload.doUpload(req, res);
+
+    const gridFs = new GridFSStorage('dataset', userId);
+
+    const upLoadResults = await Promise.all(
+      files.map((file) =>
+        gridFs.save({
+          path: file.path,
+          filename: file.originalname,
+          metadata: {
+            contentType: file.mimetype,
+            encoding: file.encoding,
+            userId
+          }
+        })
+      )
+    );
+
+    jsonRes(res, {
+      data: upLoadResults
+    });
+  } catch (error) {
+    jsonRes(res, {
+      code: 500,
+      error
+    });
+  }
+}
+
+export const config = {
+  api: {
+    bodyParser: false
+  }
+};

client/src/pages/api/user/account/update.ts

@@ -5,7 +5,7 @@ import { User } from '@/service/models/user';
 import { connectToDatabase } from '@/service/mongo';
 import { authUser } from '@/service/utils/auth';
 import { UserUpdateParams } from '@/types/user';
-import { axiosConfig, getAIChatApi, openaiBaseUrl } from '@/service/ai/openai';
+import { axiosConfig, getAIChatApi, openaiBaseUrl } from '@/service/lib/openai';
 
 /* update user info */
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {

client/src/service/errorCode.ts

@@ -39,7 +39,8 @@ export enum ERROR_ENUM {
   unAuthorization = 'unAuthorization',
   insufficientQuota = 'insufficientQuota',
   unAuthModel = 'unAuthModel',
-  unAuthKb = 'unAuthKb'
+  unAuthKb = 'unAuthKb',
+  unAuthFile = 'unAuthFile'
 }
 export const ERROR_RESPONSE: Record<
   any,
@@ -73,5 +74,11 @@ export const ERROR_RESPONSE: Record<
     statusText: ERROR_ENUM.unAuthKb,
     message: '无权使用该知识库',
     data: null
+  },
+  [ERROR_ENUM.unAuthFile]: {
+    code: 513,
+    statusText: ERROR_ENUM.unAuthFile,
+    message: '无权阅读该文件',
+    data: null
   }
 };

client/src/service/events/generateQA.ts

@@ -5,7 +5,7 @@ import { TrainingModeEnum } from '@/constants/plugin';
 import { ERROR_ENUM } from '../errorCode';
 import { sendInform } from '@/pages/api/user/inform/send';
 import { authBalanceByUid } from '../utils/auth';
-import { axiosConfig, getAIChatApi } from '../ai/openai';
+import { axiosConfig, getAIChatApi } from '../lib/openai';
 import { ChatCompletionRequestMessage } from 'openai';
 import { modelToolMap } from '@/utils/plugin';
 import { gptMessage2ChatType } from '@/utils/adapt';

client/src/service/lib/gridfs.ts

+import mongoose, { Types } from 'mongoose';
+import fs from 'fs';
+import fsp from 'fs/promises';
+import { ERROR_ENUM } from '../errorCode';
+
+enum BucketNameEnum {
+  dataset = 'dataset'
+}
+
+type FileInfo = {
+  id: string;
+  filename: string;
+  size: number;
+  contentType: string;
+  encoding: string;
+};
+
+export class GridFSStorage {
+  readonly type = 'gridfs';
+  readonly bucket: `${BucketNameEnum}`;
+  readonly uid: string;
+
+  constructor(bucket: `${BucketNameEnum}`, uid: string) {
+    this.bucket = bucket;
+    this.uid = String(uid);
+  }
+  GridFSBucket() {
+    return new mongoose.mongo.GridFSBucket(mongoose.connection.db, {
+      bucketName: this.bucket
+    });
+  }
+
+  async save({
+    path,
+    filename,
+    metadata = {}
+  }: {
+    path: string;
+    filename: string;
+    metadata?: Record<string, any>;
+  }) {
+    if (!path) return Promise.reject(`filePath is empty`);
+    if (!filename) return Promise.reject(`filename is empty`);
+
+    const stats = await fsp.stat(path);
+    if (!stats.isFile()) return Promise.reject(`${path} is not a file`);
+
+    metadata.userId = this.uid;
+    // create a gridfs bucket
+    const bucket = this.GridFSBucket();
+
+    const stream = bucket.openUploadStream(filename, {
+      metadata,
+      contentType: metadata?.contentType
+    });
+
+    // save to gridfs
+    await new Promise((resolve, reject) => {
+      fs.createReadStream(path)
+        .pipe(stream as any)
+        .on('finish', resolve)
+        .on('error', reject);
+    });
+
+    return String(stream.id);
+  }
+  async findAndAuthFile(id: string): Promise<FileInfo> {
+    if (!id) {
+      return Promise.reject(`id is empty`);
+    }
+
+    // create a gridfs bucket
+    const bucket = this.GridFSBucket();
+
+    // check if file exists
+    const files = await bucket.find({ _id: new Types.ObjectId(id) }).toArray();
+    const file = files.shift();
+    if (!file) {
+      return Promise.reject(`file not found`);
+    }
+
+    if (String(file.metadata?.userId) !== this.uid) {
+      return Promise.reject(ERROR_ENUM.unAuthFile);
+    }
+
+    return {
+      id: String(file._id),
+      filename: file.filename,
+      contentType: file.metadata?.contentType,
+      encoding: file.metadata?.encoding,
+      size: file.length
+    };
+  }
+
+  async delete(id: string) {
+    await this.findAndAuthFile(id);
+    const bucket = this.GridFSBucket();
+
+    await bucket.delete(new Types.ObjectId(id));
+    return true;
+  }
+
+  async download(id: string) {
+    await this.findAndAuthFile(id);
+
+    const bucket = this.GridFSBucket();
+
+    const stream = bucket.openDownloadStream(new Types.ObjectId(id));
+
+    const buf: Buffer = await new Promise((resolve, reject) => {
+      const buffers: Buffer[] = [];
+      stream.on('data', (data) => buffers.push(data));
+      stream.on('error', reject);
+      stream.on('end', () => resolve(Buffer.concat(buffers)));
+    });
+
+    return buf;
+  }
+}

client/src/service/moduleDispatch/agent/classifyQuestion.ts

@@ -2,7 +2,7 @@ import { adaptChatItem_openAI } from '@/utils/plugin/openai';
 import { ChatContextFilter } from '@/service/utils/chat/index';
 import type { ChatHistoryItemResType, ChatItemType } from '@/types/chat';
 import { ChatModuleEnum, ChatRoleEnum, TaskResponseKeyEnum } from '@/constants/chat';
-import { getAIChatApi, axiosConfig } from '@/service/ai/openai';
+import { getAIChatApi, axiosConfig } from '@/service/lib/openai';
 import type { ClassifyQuestionAgentItemType } from '@/types/app';
 import { countModelPrice } from '@/service/events/pushBill';
 import { UserModelSchema } from '@/types/mongoSchema';

client/src/service/moduleDispatch/agent/extract.ts

@@ -2,7 +2,7 @@ import { adaptChatItem_openAI } from '@/utils/plugin/openai';
 import { ChatContextFilter } from '@/service/utils/chat/index';
 import type { ChatHistoryItemResType, ChatItemType } from '@/types/chat';
 import { ChatModuleEnum, ChatRoleEnum, TaskResponseKeyEnum } from '@/constants/chat';
-import { getAIChatApi, axiosConfig } from '@/service/ai/openai';
+import { getAIChatApi, axiosConfig } from '@/service/lib/openai';
 import type { ContextExtractAgentItemType } from '@/types/app';
 import { ContextExtractEnum } from '@/constants/flow/flowField';
 import { countModelPrice } from '@/service/events/pushBill';

client/src/service/moduleDispatch/chat/oneapi.ts

@@ -8,7 +8,7 @@ import type { ChatHistoryItemResType } from '@/types/chat';
 import { ChatModuleEnum, ChatRoleEnum, sseResponseEventEnum } from '@/constants/chat';
 import { SSEParseData, parseStreamChunk } from '@/utils/sse';
 import { textAdaptGptResponse } from '@/utils/adapt';
-import { getAIChatApi, axiosConfig } from '@/service/ai/openai';
+import { getAIChatApi, axiosConfig } from '@/service/lib/openai';
 import { TaskResponseKeyEnum } from '@/constants/chat';
 import { getChatModel } from '@/service/utils/data';
 import { countModelPrice } from '@/service/events/pushBill';

client/src/service/plugins/useSearch.ts

-import axios from 'axios';
-
-{
-  /*Bing 搜索*/
-}
-const BingSearch = async (wait_val: string) => {
-  const response = await axios.post('newbing中转服务器', {
-    prompt: wait_val
-  });
-  const result = response.data.result;
-  return result;
-};
-
-{
-  /*google 搜索*/
-}
-const GoogleSearch = async (wait_val: string) => {
-  const response = await axios.get('https://www.googleapis.com/customsearch/v1', {
-    params: {
-      key: process.env.GOOGLE_KEY,
-      q: wait_val,
-      cx: process.env.searchEngineId,
-      start: 1,
-      num: 3,
-      dateRestrict: 'm[1]' //搜索结果限定为一个月内
-    }
-  });
-  const results = response.data.items;
-  if (results !== null) {
-    const result = results.map((item: { snippet: string }) => item.snippet).join('\n');
-    return result;
-  }
-};
-export { BingSearch, GoogleSearch };