fix: tiktoken memory

package.json

@@ -55,7 +55,6 @@
     "remark-math": "^5.1.1",
     "request-ip": "^3.3.0",
     "sass": "^1.58.3",
-    "sharp": "^0.31.3",
     "tunnel": "^0.0.6",
     "wxpay-v3": "^3.0.2",
     "zustand": "^4.3.5"

src/service/mongo.ts

@@ -37,6 +37,7 @@ export async function connectToDatabase(): Promise<void> {
     });
   }
 
+  // 初始化队列
   global.qaQueueLen = 0;
   global.vectorQueueLen = 0;
 

src/types/index.d.ts

 import type { Mongoose } from 'mongoose';
 import type { Agent } from 'http';
 import type { Pool } from 'pg';
+import type { Tiktoken } from '@dqbd/tiktoken';
 
 declare global {
   var mongodb: Mongoose | string | null;
@@ -11,6 +12,7 @@ declare global {
   var QRCode: any;
   var qaQueueLen: number;
   var vectorQueueLen: number;
+  var OpenAiEncMap: Record<string, Tiktoken>;
 
   interface Window {
     ['pdfjs-dist/build/pdf']: any;

src/utils/plugin/openai.ts

@@ -8,27 +8,48 @@ import Graphemer from 'graphemer';
 const textDecoder = new TextDecoder();
 const graphemer = new Graphemer();
 
-export const adaptChatItem_openAI = ({
-  messages
-}: {
-  messages: ChatItemSimpleType[];
-}): ChatCompletionRequestMessage[] => {
-  const map = {
-    [ChatRoleEnum.AI]: ChatCompletionRequestMessageRoleEnum.Assistant,
-    [ChatRoleEnum.Human]: ChatCompletionRequestMessageRoleEnum.User,
-    [ChatRoleEnum.System]: ChatCompletionRequestMessageRoleEnum.System
-  };
-  return messages.map((item) => ({
-    role: map[item.obj] || ChatCompletionRequestMessageRoleEnum.System,
-    content: item.value || ''
-  }));
-};
-
-/* count openai chat token*/
-let OpenAiEncMap: Record<string, Tiktoken>;
 export const getOpenAiEncMap = () => {
-  if (OpenAiEncMap) return OpenAiEncMap;
-  OpenAiEncMap = {
+  if (typeof window !== 'undefined') {
+    window.OpenAiEncMap = window.OpenAiEncMap || {
+      'gpt-3.5-turbo': encoding_for_model('gpt-3.5-turbo', {
+        '<|im_start|>': 100264,
+        '<|im_end|>': 100265,
+        '<|im_sep|>': 100266
+      }),
+      'gpt-4': encoding_for_model('gpt-4', {
+        '<|im_start|>': 100264,
+        '<|im_end|>': 100265,
+        '<|im_sep|>': 100266
+      }),
+      'gpt-4-32k': encoding_for_model('gpt-4-32k', {
+        '<|im_start|>': 100264,
+        '<|im_end|>': 100265,
+        '<|im_sep|>': 100266
+      })
+    };
+    return window.OpenAiEncMap;
+  }
+  if (typeof global !== 'undefined') {
+    global.OpenAiEncMap = global.OpenAiEncMap || {
+      'gpt-3.5-turbo': encoding_for_model('gpt-3.5-turbo', {
+        '<|im_start|>': 100264,
+        '<|im_end|>': 100265,
+        '<|im_sep|>': 100266
+      }),
+      'gpt-4': encoding_for_model('gpt-4', {
+        '<|im_start|>': 100264,
+        '<|im_end|>': 100265,
+        '<|im_sep|>': 100266
+      }),
+      'gpt-4-32k': encoding_for_model('gpt-4-32k', {
+        '<|im_start|>': 100264,
+        '<|im_end|>': 100265,
+        '<|im_sep|>': 100266
+      })
+    };
+    return global.OpenAiEncMap;
+  }
+  return {
     'gpt-3.5-turbo': encoding_for_model('gpt-3.5-turbo', {
       '<|im_start|>': 100264,
       '<|im_end|>': 100265,
@@ -45,8 +66,24 @@ export const getOpenAiEncMap = () => {
       '<|im_sep|>': 100266
     })
   };
-  return OpenAiEncMap;
 };
+
+export const adaptChatItem_openAI = ({
+  messages
+}: {
+  messages: ChatItemSimpleType[];
+}): ChatCompletionRequestMessage[] => {
+  const map = {
+    [ChatRoleEnum.AI]: ChatCompletionRequestMessageRoleEnum.Assistant,
+    [ChatRoleEnum.Human]: ChatCompletionRequestMessageRoleEnum.User,
+    [ChatRoleEnum.System]: ChatCompletionRequestMessageRoleEnum.System
+  };
+  return messages.map((item) => ({
+    role: map[item.obj] || ChatCompletionRequestMessageRoleEnum.System,
+    content: item.value || ''
+  }));
+};
+
 export function countOpenAIToken({
   messages,
   model