perf: chunk filter

client/src/pages/kb/detail/components/Import/Chunk.tsx

@@ -16,7 +16,7 @@ import { useConfirm } from '@/hooks/useConfirm';
 import { readTxtContent, readPdfContent, readDocContent } from '@/utils/file';
 import { useMutation } from '@tanstack/react-query';
 import { postKbDataFromList } from '@/api/plugins/kb';
-import { splitText_token } from '@/utils/file';
+import { splitText2Chunks } from '@/utils/file';
 import { getErrText } from '@/utils/tools';
 import { formatPrice } from '@/utils/user';
 import { vectorModelList } from '@/store/static';
@@ -96,7 +96,7 @@ const ChunkImport = ({ kbId }: { kbId: string }) => {
             })();
 
             if (icon && text) {
-              const splitRes = splitText_token({
+              const splitRes = splitText2Chunks({
                 text: text,
                 maxLen: chunkLen
               });
@@ -178,7 +178,7 @@ const ChunkImport = ({ kbId }: { kbId: string }) => {
   const onRePreview = useCallback(async () => {
     try {
       const splitRes = files.map((item) =>
-        splitText_token({
+        splitText2Chunks({
           text: item.text,
           maxLen: chunkLen
         })

client/src/pages/kb/detail/components/Import/QA.tsx

@@ -5,7 +5,7 @@ import { useConfirm } from '@/hooks/useConfirm';
 import { readTxtContent, readPdfContent, readDocContent } from '@/utils/file';
 import { useMutation } from '@tanstack/react-query';
 import { postKbDataFromList } from '@/api/plugins/kb';
-import { splitText_token } from '@/utils/file';
+import { splitText2Chunks } from '@/utils/file';
 import { getErrText } from '@/utils/tools';
 import { formatPrice } from '@/utils/user';
 import { qaModelList } from '@/store/static';
@@ -86,7 +86,7 @@ const QAImport = ({ kbId }: { kbId: string }) => {
             })();
 
             if (icon && text) {
-              const splitRes = splitText_token({
+              const splitRes = splitText2Chunks({
                 text: text,
                 maxLen: chunkLen
               });
@@ -169,7 +169,7 @@ const QAImport = ({ kbId }: { kbId: string }) => {
   const onRePreview = useCallback(async () => {
     try {
       const splitRes = files.map((item) =>
-        splitText_token({
+        splitText2Chunks({
           text: item.text,
           maxLen: chunkLen
         })

client/src/utils/file.ts

@@ -147,7 +147,7 @@ export const fileDownload = ({
  * overlapLen - The size of the before and after Text
  * maxLen > overlapLen
  */
-export const splitText_token = ({ text, maxLen }: { text: string; maxLen: number }) => {
+export const splitText2Chunks = ({ text, maxLen }: { text: string; maxLen: number }) => {
   const overlapLen = Math.floor(maxLen * 0.3); // Overlap length
 
   try {