fix: chat variable update (#3156)

* perf: file encoding

* fix: chat variable update

packages/service/common/file/gridfs/controller.ts

@@ -36,6 +36,7 @@ export async function uploadFile({
   path,
   path,
   filename,
   filename,
   contentType,
   contentType,
+  encoding,
   metadata = {}
   metadata = {}
 }: {
 }: {
   bucketName: `${BucketNameEnum}`;
   bucketName: `${BucketNameEnum}`;
@@ -44,6 +45,7 @@ export async function uploadFile({
   path: string;
   path: string;
   filename: string;
   filename: string;
   contentType?: string;
   contentType?: string;
+  encoding: string;
   metadata?: Record<string, any>;
   metadata?: Record<string, any>;
 }) {
 }) {
   if (!path) return Promise.reject(`filePath is empty`);
   if (!path) return Promise.reject(`filePath is empty`);
@@ -52,7 +54,7 @@ export async function uploadFile({
   const stats = await fsp.stat(path);
   const stats = await fsp.stat(path);
   if (!stats.isFile()) return Promise.reject(`${path} is not a file`);
   if (!stats.isFile()) return Promise.reject(`${path} is not a file`);
 
 
-  const { stream: readStream, encoding } = await stream2Encoding(fs.createReadStream(path));
+  const readStream = fs.createReadStream(path);
 
 
   // Add default metadata
   // Add default metadata
   metadata.teamId = teamId;
   metadata.teamId = teamId;

packages/service/common/file/read/utils.ts

@@ -14,6 +14,7 @@ import { addHours } from 'date-fns';
 export type readRawTextByLocalFileParams = {
 export type readRawTextByLocalFileParams = {
   teamId: string;
   teamId: string;
   path: string;
   path: string;
+  encoding: string;
   metadata?: Record<string, any>;
   metadata?: Record<string, any>;
 };
 };
 export const readRawTextByLocalFile = async (params: readRawTextByLocalFileParams) => {
 export const readRawTextByLocalFile = async (params: readRawTextByLocalFileParams) => {
@@ -22,13 +23,12 @@ export const readRawTextByLocalFile = async (params: readRawTextByLocalFileParam
   const extension = path?.split('.')?.pop()?.toLowerCase() || '';
   const extension = path?.split('.')?.pop()?.toLowerCase() || '';
 
 
   const buffer = fs.readFileSync(path);
   const buffer = fs.readFileSync(path);
-  const encoding = detectFileEncoding(buffer);
 
 
   const { rawText } = await readRawContentByFileBuffer({
   const { rawText } = await readRawContentByFileBuffer({
     extension,
     extension,
     isQAImport: false,
     isQAImport: false,
     teamId: params.teamId,
     teamId: params.teamId,
-    encoding,
+    encoding: params.encoding,
     buffer,
     buffer,
     metadata: params.metadata
     metadata: params.metadata
   });
   });
@@ -53,6 +53,7 @@ export const readRawContentByFileBuffer = async ({
   encoding: string;
   encoding: string;
   metadata?: Record<string, any>;
   metadata?: Record<string, any>;
 }) => {
 }) => {
+  // Custom read file service
   const customReadfileUrl = process.env.CUSTOM_READ_FILE_URL;
   const customReadfileUrl = process.env.CUSTOM_READ_FILE_URL;
   const customReadFileExtension = process.env.CUSTOM_READ_FILE_EXTENSION || '';
   const customReadFileExtension = process.env.CUSTOM_READ_FILE_EXTENSION || '';
   const ocrParse = process.env.CUSTOM_READ_FILE_OCR || 'false';
   const ocrParse = process.env.CUSTOM_READ_FILE_OCR || 'false';

packages/service/worker/readFile/extension/rawText.ts

@@ -18,9 +18,17 @@ const rawEncodingList = [
 
 
 // 加载源文件内容
 // 加载源文件内容
 export const readFileRawText = ({ buffer, encoding }: ReadRawTextByBuffer): ReadFileResponse => {
 export const readFileRawText = ({ buffer, encoding }: ReadRawTextByBuffer): ReadFileResponse => {
-  const content = rawEncodingList.includes(encoding)
-    ? buffer.toString(encoding as BufferEncoding)
-    : iconv.decode(buffer, 'gbk');
+  const content = (() => {
+    try {
+      if (rawEncodingList.includes(encoding)) {
+        return buffer.toString(encoding as BufferEncoding);
+      }
+
+      return iconv.decode(buffer, encoding);
+    } catch (error) {
+      return buffer.toString('utf-8');
+    }
+  })();
 
 
   return {
   return {
     rawText: content
     rawText: content

projects/app/src/components/core/chat/ChatContainer/ChatBox/index.tsx

@@ -323,7 +323,7 @@ const ChatBox = (
               })
               })
             };
             };
           } else if (event === SseResponseEventEnum.updateVariables && variables) {
           } else if (event === SseResponseEventEnum.updateVariables && variables) {
-            variablesForm.reset(variables);
+            variablesForm.setValue('variables', variables);
           } else if (event === SseResponseEventEnum.interactive) {
           } else if (event === SseResponseEventEnum.interactive) {
             const val: AIChatItemValueItemType = {
             const val: AIChatItemValueItemType = {
               type: ChatItemValueTypeEnum.interactive,
               type: ChatItemValueTypeEnum.interactive,
@@ -408,7 +408,7 @@ const ChatBox = (
       isInteractivePrompt = false
       isInteractivePrompt = false
     }) => {
     }) => {
       variablesForm.handleSubmit(
       variablesForm.handleSubmit(
-        async ({ variables }) => {
+        async ({ variables = {} }) => {
           if (!onStartChat) return;
           if (!onStartChat) return;
           if (isChatting) {
           if (isChatting) {
             toast({
             toast({
@@ -435,7 +435,7 @@ const ChatBox = (
           // Only declared variables are kept
           // Only declared variables are kept
           const requestVariables: Record<string, any> = {};
           const requestVariables: Record<string, any> = {};
           allVariableList?.forEach((item) => {
           allVariableList?.forEach((item) => {
-            requestVariables[item.key] = variables[item.key] || '';
+            requestVariables[item.key] = variables[item.key];
           });
           });
 
 
           const responseChatId = getNanoid(24);
           const responseChatId = getNanoid(24);

projects/app/src/pages/api/common/file/upload.ts

@@ -51,6 +51,7 @@ async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
       path: file.path,
       path: file.path,
       filename: file.originalname,
       filename: file.originalname,
       contentType: file.mimetype,
       contentType: file.mimetype,
+      encoding: file.encoding,
       metadata: metadata
       metadata: metadata
     });
     });
 
 

projects/app/src/pages/api/core/dataset/collection/create/localFile.ts

@@ -67,6 +67,7 @@ async function handler(req: NextApiRequest, res: NextApiResponse<any>): CreateCo
     const { rawText } = await readRawTextByLocalFile({
     const { rawText } = await readRawTextByLocalFile({
       teamId,
       teamId,
       path: file.path,
       path: file.path,
+      encoding: file.encoding,
       metadata: {
       metadata: {
         ...fileMetadata,
         ...fileMetadata,
         relatedId: relatedImgId
         relatedId: relatedImgId
@@ -81,6 +82,7 @@ async function handler(req: NextApiRequest, res: NextApiResponse<any>): CreateCo
       path: file.path,
       path: file.path,
       filename: file.originalname,
       filename: file.originalname,
       contentType: file.mimetype,
       contentType: file.mimetype,
+      encoding: file.encoding,
       metadata: fileMetadata
       metadata: fileMetadata
     });
     });
 
 

projects/app/src/pages/dataset/detail/components/Import/Context.tsx

@@ -254,8 +254,6 @@ const DatasetImportContextProvider = ({ children }: { children: React.ReactNode
               icon={<MyIcon name={'common/backFill'} w={'14px'} />}
               icon={<MyIcon name={'common/backFill'} w={'14px'} />}
               aria-label={''}
               aria-label={''}
               size={'smSquare'}
               size={'smSquare'}
-              w={'26px'}
-              h={'26px'}
               borderRadius={'50%'}
               borderRadius={'50%'}
               variant={'whiteBase'}
               variant={'whiteBase'}
               mr={2}
               mr={2}