perf: token count

client/src/pages/api/openapi/kb/pushData.ts

@@ -76,14 +76,17 @@ export async function pushDataToKb({
   data.forEach((item) => {
   data.forEach((item) => {
     const text = item.q + item.a;
     const text = item.q + item.a;
 
 
-    // count token
-    const token = modelToolMap[OpenAiChatEnum.GPT35].countTokens({
-      messages: [{ obj: 'System', value: item.q }]
-    });
+    if (mode === TrainingModeEnum.qa) {
+      // count token
+      const token = modelToolMap[OpenAiChatEnum.GPT35].countTokens({
+        messages: [{ obj: 'System', value: item.q }]
+      });
+      if (token > modeMaxToken[TrainingModeEnum.qa]) {
+        return;
+      }
+    }
 
 
-    if (mode === TrainingModeEnum.qa && token > modeMaxToken[TrainingModeEnum.qa]) {
-      console.log('q is too long');
-    } else if (!set.has(text)) {
+    if (!set.has(text)) {
       filterData.push(item);
       filterData.push(item);
       set.add(text);
       set.add(text);
     }
     }

client/src/pages/kb/components/Detail.tsx

@@ -40,6 +40,7 @@ const Detail = ({ kbId }: { kbId: string }) => {
     onSuccess(res) {
     onSuccess(res) {
       kbId && setLastKbId(kbId);
       kbId && setLastKbId(kbId);
       if (res) {
       if (res) {
+        setCurrentTab(TabEnum.data);
         reset(res);
         reset(res);
         BasicInfo.current?.initInput?.(res.tags);
         BasicInfo.current?.initInput?.(res.tags);
       }
       }

client/src/service/pg.ts

@@ -12,9 +12,9 @@ export const connectPg = async () => {
     user: process.env.PG_USER,
     user: process.env.PG_USER,
     password: process.env.PG_PASSWORD,
     password: process.env.PG_PASSWORD,
     database: process.env.PG_DB_NAME,
     database: process.env.PG_DB_NAME,
-    max: 80,
-    idleTimeoutMillis: 60000,
-    connectionTimeoutMillis: 20000
+    max: global.systemEnv.vectorMaxProcess + 10,
+    idleTimeoutMillis: 30000,
+    connectionTimeoutMillis: 5000
   });
   });
 
 
   global.pgClient.on('error', (err) => {
   global.pgClient.on('error', (err) => {

client/src/service/utils/chat/claude.ts

@@ -24,7 +24,7 @@ export const claudChat = async ({ apiKey, messages, stream, chatId }: ChatComple
       headers: {
       headers: {
         Authorization: apiKey
         Authorization: apiKey
       },
       },
-      timeout: stream ? 60000 : 240000,
+      timeout: stream ? 60000 : 480000,
       responseType: stream ? 'stream' : 'json'
       responseType: stream ? 'stream' : 'json'
     }
     }
   );
   );

client/src/service/utils/chat/openai.ts

@@ -42,7 +42,7 @@ export const chatResponse = async ({
       stop: ['.!?。']
       stop: ['.!?。']
     },
     },
     {
     {
-      timeout: stream ? 60000 : 240000,
+      timeout: stream ? 60000 : 480000,
       responseType: stream ? 'stream' : 'json',
       responseType: stream ? 'stream' : 'json',
       ...axiosConfig(apiKey)
       ...axiosConfig(apiKey)
     }
     }