修改插件：Doc2X插件适配文件上传功能 (#2284)

* Change to download first and check contentType

* Up to date

* Some bug fix, still some bug with img

* Update tool to read from file

* improve formatting in PDF

* Add tool of img file OCR , but meet some bug

* Bug fix for parameter passing error.

* Modification Introduction

packages/plugins/register.ts

@@ -13,6 +13,8 @@ const staticPluginList = [
   'Doc2X',
   'Doc2X/URLPDF2text',
   'Doc2X/URLImg2text',
+  `Doc2X/FilePDF2text`,
+  `Doc2X/FileImg2text`,
   'feishu'
 ];
 // Run in worker thread (Have npm packages)

packages/plugins/src/Doc2X/FileImg2text/index.ts

+import { delay } from '@fastgpt/global/common/system/utils';
+import { addLog } from '@fastgpt/service/common/system/log';
+
+type Props = {
+  apikey: string;
+  files: Array<string>;
+  img_correction: boolean;
+  formula: boolean;
+};
+
+type Response = Promise<{
+  result: string;
+  failreason: string;
+  success: boolean;
+}>;
+
+const main = async ({ apikey, files, img_correction, formula }: Props): Response => {
+  // Check the apikey
+  if (!apikey) {
+    return {
+      result: '',
+      failreason: `API key is required`,
+      success: false
+    };
+  }
+
+  let real_api_key = apikey;
+  if (!apikey.startsWith('sk-')) {
+    const response = await fetch('https://api.doc2x.noedgeai.com/api/token/refresh', {
+      method: 'POST',
+      headers: {
+        Authorization: `Bearer ${apikey}`
+      }
+    });
+    if (response.status !== 200) {
+      return {
+        result: '',
+        failreason: `Get token failed: ${await response.text()}`,
+        success: false
+      };
+    }
+    const data = await response.json();
+    real_api_key = data.data.token;
+  }
+
+  let final_result = '';
+  let fail_reason = '';
+  let flag = false;
+  //Process each file one by one
+  for await (const url of files) {
+    // Fetch the image and check its content type
+    const imageResponse = await fetch(url);
+    if (!imageResponse.ok) {
+      fail_reason += `\n---\nFile:${url} \n<Content>\nFailed to fetch image from URL\n</Content>\n`;
+      flag = true;
+      continue;
+    }
+
+    const contentType = imageResponse.headers.get('content-type');
+    const fileName = url.match(/read\?filename=([^&]+)/)?.[1] || 'unknown.png';
+    if (!contentType || !contentType.startsWith('image/')) {
+      fail_reason += `\n---\nFile:${url} \n<Content>\nThe provided URL does not point to an image: ${contentType}\n</Content>\n`;
+      flag = true;
+      continue;
+    }
+
+    const blob = await imageResponse.blob();
+    const formData = new FormData();
+    formData.append('file', blob, fileName);
+    formData.append('img_correction', img_correction ? '1' : '0');
+    formData.append('equation', formula ? '1' : '0');
+
+    let upload_url = 'https://api.doc2x.noedgeai.com/api/platform/async/img';
+    if (real_api_key.startsWith('sk-')) {
+      upload_url = 'https://api.doc2x.noedgeai.com/api/v1/async/img';
+    }
+
+    let uuid;
+    let upload_flag = true;
+    const uploadAttempts = [1, 2, 3];
+    for await (const attempt of uploadAttempts) {
+      const upload_response = await fetch(upload_url, {
+        method: 'POST',
+        headers: {
+          Authorization: `Bearer ${real_api_key}`
+        },
+        body: formData
+      });
+
+      if (!upload_response.ok) {
+        // Rate limit, wait for 10s and retry at most 3 times
+        if (upload_response.status === 429 && attempt < 3) {
+          await delay(10000);
+          continue;
+        }
+        fail_reason += `\n---\nFile:${fileName}\n<Content>\nFailed to upload file: ${await upload_response.text()}\n</Content>\n`;
+        flag = true;
+        upload_flag = false;
+        break;
+      }
+      if (!upload_flag) {
+        continue;
+      }
+
+      const upload_data = await upload_response.json();
+      uuid = upload_data.data.uuid;
+      break;
+    }
+
+    // Get the result by uuid
+    let result_url = 'https://api.doc2x.noedgeai.com/api/platform/async/status?uuid=' + uuid;
+    if (real_api_key.startsWith('sk-')) {
+      result_url = 'https://api.doc2x.noedgeai.com/api/v1/async/status?uuid=' + uuid;
+    }
+
+    let required_flag = true;
+    const maxAttempts = 100;
+    // Wait for the result, at most 100s
+    for await (const _ of Array(maxAttempts).keys()) {
+      const result_response = await fetch(result_url, {
+        headers: {
+          Authorization: `Bearer ${real_api_key}`
+        }
+      });
+      if (!result_response.ok) {
+        fail_reason += `\n---\nFile:${fileName}\n<Content>\nFailed to get result: ${await result_response.text()}\n</Content>\n`;
+        flag = true;
+        required_flag = false;
+        break;
+      }
+      const result_data = await result_response.json();
+      if (['ready', 'processing'].includes(result_data.data.status)) {
+        await delay(1000);
+      } else if (result_data.data.status === 'pages limit exceeded') {
+        fail_reason += `\n---\nFile:${fileName}\n<Content>\nFailed to get result: pages limit exceeded\n</Content>\n`;
+        flag = true;
+        required_flag = false;
+        break;
+      } else if (result_data.data.status === 'success') {
+        let result;
+        try {
+          result = result_data.data.result.pages[0].md;
+          result = result.replace(/\\[\(\)]/g, '$').replace(/\\[\[\]]/g, '$$');
+        } catch {
+          // no pages
+          final_result += `\n---\nFile:${fileName}\n<Content>\n \n</Content>\n`;
+          required_flag = false;
+        }
+        final_result += `\n---\nFile:${fileName}\n<Content>\n${result}\n</Content>\n`;
+        required_flag = false;
+        break;
+      } else {
+        fail_reason += `\n---\nFile:${fileName}\n<Content>\nFailed to get result: ${result_data.data.status}\n</Content>\n`;
+        flag = true;
+        required_flag = false;
+        break;
+      }
+    }
+    if (required_flag) {
+      fail_reason += `\n---\nFile:${fileName}\n<Content>\nTimeout waiting for result\n</Content>\n`;
+      flag = true;
+    }
+  }
+
+  return {
+    result: final_result,
+    failreason: fail_reason,
+    success: !flag
+  };
+};
+
+export default main;

packages/plugins/src/Doc2X/FilePDF2text/index.ts

+import { delay } from '@fastgpt/global/common/system/utils';
+import { addLog } from '@fastgpt/service/common/system/log';
+import { result } from 'lodash';
+
+type Props = {
+  apikey: string;
+  files: Array<string>;
+  ocr: boolean;
+};
+
+// Response type same as HTTP outputs
+type Response = Promise<{
+  result: string;
+  failreason: string;
+  success: boolean;
+}>;
+
+const main = async ({ apikey, files, ocr }: Props): Response => {
+  // Check the apikey
+  if (!apikey) {
+    return {
+      result: '',
+      failreason: `API key is required`,
+      success: false
+    };
+  }
+
+  let real_api_key = apikey;
+  if (!apikey.startsWith('sk-')) {
+    const response = await fetch('https://api.doc2x.noedgeai.com/api/token/refresh', {
+      method: 'POST',
+      headers: {
+        Authorization: `Bearer ${apikey}`
+      }
+    });
+    if (response.status !== 200) {
+      return {
+        result: '',
+        failreason: `Get token failed: ${await response.text()}`,
+        success: false
+      };
+    }
+    const data = await response.json();
+    real_api_key = data.data.token;
+  }
+
+  let final_result = '';
+  let fail_reason = '';
+  let flag = false;
+  //Process each file one by one
+  for await (const url of files) {
+    //Fetch the pdf and check its contene type
+    const PDFResponse = await fetch(url);
+    if (!PDFResponse.ok) {
+      fail_reason += `\n---\nFile:${url} \n<Content>\nFailed to fetch PDF from URL\n</Content>\n`;
+      flag = true;
+      continue;
+    }
+
+    const contentType = PDFResponse.headers.get('content-type');
+    const file_name = url.match(/read\?filename=([^&]+)/)?.[1] || 'unknown.pdf';
+    if (!contentType || !contentType.startsWith('application/pdf')) {
+      fail_reason += `\n---\nFile:${file_name}\n<Content>\nThe provided file does not point to a PDF: ${contentType}\n</Content>\n`;
+      flag = true;
+      continue;
+    }
+
+    const blob = await PDFResponse.blob();
+    const formData = new FormData();
+    formData.append('file', blob, file_name);
+    formData.append('ocr', ocr ? '1' : '0');
+
+    let upload_url = 'https://api.doc2x.noedgeai.com/api/platform/async/pdf';
+    if (real_api_key.startsWith('sk-')) {
+      upload_url = 'https://api.doc2x.noedgeai.com/api/v1/async/pdf';
+    }
+
+    let uuid;
+    let upload_flag = true;
+    const uploadAttempts = [1, 2, 3];
+    for await (const attempt of uploadAttempts) {
+      const upload_response = await fetch(upload_url, {
+        method: 'POST',
+        headers: {
+          Authorization: `Bearer ${real_api_key}`
+        },
+        body: formData
+      });
+      if (!upload_response.ok) {
+        // Rate limit, wait for 10s and retry at most 3 times
+        if (upload_response.status === 429 && attempt < 3) {
+          await delay(10000);
+          continue;
+        }
+        fail_reason += `\n---\nFile:${file_name}\n<Content>\nFailed to upload file: ${await upload_response.text()}\n</Content>\n`;
+        flag = true;
+        upload_flag = false;
+      }
+      if (!upload_flag) {
+        continue;
+      }
+      const upload_data = await upload_response.json();
+      uuid = upload_data.data.uuid;
+      break;
+    }
+
+    // Get the result by uuid
+    let result_url = 'https://api.doc2x.noedgeai.com/api/platform/async/status?uuid=' + uuid;
+    if (real_api_key.startsWith('sk-')) {
+      result_url = 'https://api.doc2x.noedgeai.com/api/v1/async/status?uuid=' + uuid;
+    }
+
+    let required_flag = true;
+    let result = '';
+    // Wait for the result, at most 100s
+    const maxAttempts = 100;
+    for await (const _ of Array(maxAttempts).keys()) {
+      const result_response = await fetch(result_url, {
+        headers: {
+          Authorization: `Bearer ${real_api_key}`
+        }
+      });
+      if (!result_response.ok) {
+        fail_reason += `\n---\nFile:${file_name}\n<Content>\nFailed to get result: ${await result_response.text()}\n</Content>\n`;
+        flag = true;
+        required_flag = false;
+        break;
+      }
+      const result_data = await result_response.json();
+      if (['ready', 'processing'].includes(result_data.data.status)) {
+        await delay(1000);
+      } else if (result_data.data.status === 'pages limit exceeded') {
+        fail_reason += `\n---\nFile:${file_name}\n<Content>\nPages limit exceeded\n</Content>\n`;
+        flag = true;
+        required_flag = false;
+        break;
+      } else if (result_data.data.status === 'success') {
+        result = await Promise.all(
+          result_data.data.result.pages.map((page: { md: any }) => page.md)
+        ).then((pages) => pages.join('\n'));
+        result = result.replace(/\\[\(\)]/g, '$').replace(/\\[\[\]]/g, '$$');
+        final_result += `\n---\nFile:${file_name}\n<Content>\n${result}\n</Content>\n`;
+        required_flag = false;
+        break;
+      } else {
+        fail_reason += `\n---\nFile:${file_name}\n<Content>\nFailed to get result: ${result_data.data.status}\n</Content>\n`;
+        flag = true;
+        required_flag = false;
+        break;
+      }
+    }
+    if (required_flag) {
+      fail_reason += `\n---\nFile:${file_name}\n<Content>\nTimeout after 100s for uuid ${uuid}\n</Content>\n`;
+      flag = true;
+    }
+  }
+
+  return {
+    result: final_result,
+    failreason: fail_reason,
+    success: !flag
+  };
+};
+
+export default main;

packages/plugins/src/Doc2X/URLImg2text/index.ts

@@ -8,7 +8,6 @@ type Props = {
   formula: boolean;
 };
 
-// Response type same as HTTP outputs
 type Response = Promise<{
   result: string;
   success: boolean;
@@ -41,36 +40,36 @@ const main = async ({ apikey, url, img_correction, formula }: Props): Response =
     real_api_key = data.data.token;
   }
 
-  //Get the image binary from the URL
-  const extension = url.split('.').pop()?.toLowerCase();
-  const name = url.split('/').pop()?.split('.').shift();
-  let mini = '';
-  switch (extension) {
-    case 'jpg':
-    case 'jpeg':
-      mini = 'image/jpeg';
-      break;
-    case 'png':
-      mini = 'image/png';
-      break;
-    default:
-      return {
-        result: `Not supported image format, only support jpg/jpeg/png`,
-        success: false
-      };
+  let imageResponse;
+  // Fetch the image and check its content type
+  try {
+    imageResponse = await fetch(url);
+  } catch (e) {
+    return {
+      result: `Failed to fetch image from URL: ${url} with error: ${e}`,
+      success: false
+    };
   }
 
-  const response = await fetch(url);
-  if (!response.ok) {
+  if (!imageResponse.ok) {
     return {
       result: `Failed to fetch image from URL: ${url}`,
       success: false
     };
   }
 
-  const blob = await response.blob();
+  const contentType = imageResponse.headers.get('content-type');
+  if (!contentType || !contentType.startsWith('image/')) {
+    return {
+      result: `The provided URL does not point to an image: ${contentType}`,
+      success: false
+    };
+  }
+
+  const blob = await imageResponse.blob();
   const formData = new FormData();
-  formData.append('file', new Blob([blob], { type: mini }), name + '.' + extension);
+  const fileName = url.split('/').pop()?.split('?')[0] || 'image';
+  formData.append('file', blob, fileName);
   formData.append('img_correction', img_correction ? '1' : '0');
   formData.append('equation', formula ? '1' : '0');
 
@@ -135,8 +134,17 @@ const main = async ({ apikey, url, img_correction, formula }: Props): Response =
         success: false
       };
     } else if (result_data.data.status === 'success') {
-      let result = result_data.data.result.pages[0].md;
-      result = result.replace(/\\[\(\)]/g, '$').replace(/\\[\[\]]/g, '$$');
+      let result;
+      try {
+        result = result_data.data.result.pages[0].md;
+        result = result.replace(/\\[\(\)]/g, '$').replace(/\\[\[\]]/g, '$$');
+      } catch {
+        // no pages
+        return {
+          result: '',
+          success: true
+        };
+      }
       return {
         result: result,
         success: true

packages/plugins/src/Doc2X/URLImg2text/template.json

@@ -3,7 +3,7 @@
   "version": "488",
   "name": "Doc2X 图像(URL)识别",
   "avatar": "plugins/doc2x",
-  "intro": "将传入的图片(URL)发送至Doc2X进行解析，返回带LaTeX公式的markdown格式的文本",
+  "intro": "从URL下载图片并发送至Doc2X进行解析，返回带LaTeX公式的markdown格式的文本",
   "inputExplanationUrl": "https://fael3z0zfze.feishu.cn/wiki/Rkc5witXWiJoi5kORd2cofh6nDg?fromScene=spaceOverview",
   "showStatus": true,
   "weight": 10,
@@ -220,7 +220,7 @@
             "key": "system_httpJsonBody",
             "renderTypeList": ["hidden"],
             "valueType": "any",
-            "value": "{\n  \"apikey\": \"{{apikey}}\",\n  \"url\": \"{{url}}\",\n  \"img_correction\": \"{{img_correction}}\",\n  \"formula\": \"{{img_correction}}\"\n}",
+            "value": "{\n  \"apikey\": \"{{apikey}}\",\n  \"url\": \"{{url}}\",\n  \"img_correction\": {{img_correction}},\n  \"formula\": {{formula}}\n}",
             "label": "",
             "required": false
           },

packages/plugins/src/Doc2X/URLPDF2text/index.ts

@@ -40,9 +40,35 @@ const main = async ({ apikey, url, ocr }: Props): Response => {
     real_api_key = data.data.token;
   }
 
-  //Get the image binary from the URL
+  //Fetch the pdf and check its contene type
+  let PDFResponse;
+  try {
+    PDFResponse = await fetch(url);
+  } catch (e) {
+    return {
+      result: `Failed to fetch PDF from URL: ${url} with error: ${e}`,
+      success: false
+    };
+  }
+  if (!PDFResponse.ok) {
+    return {
+      result: `Failed to fetch PDF from URL: ${url}`,
+      success: false
+    };
+  }
+
+  const contentType = PDFResponse.headers.get('content-type');
+  if (!contentType || !contentType.startsWith('application/pdf')) {
+    return {
+      result: `The provided URL does not point to a PDF: ${contentType}`,
+      success: false
+    };
+  }
+
+  const blob = await PDFResponse.blob();
   const formData = new FormData();
-  formData.append('pdf_url', url);
+  const fileName = url.split('/').pop()?.split('?')[0] || 'pdf';
+  formData.append('file', blob, fileName);
   formData.append('ocr', ocr ? '1' : '0');
 
   let upload_url = 'https://api.doc2x.noedgeai.com/api/platform/async/pdf';

packages/plugins/src/Doc2X/URLPDF2text/template.json

@@ -3,7 +3,7 @@
   "version": "488",
   "name": "Doc2X PDF文件(URL)识别",
   "avatar": "plugins/doc2x",
-  "intro": "将传入的PDF文件(URL)发送至Doc2X进行解析，返回带LaTeX公式的markdown格式的文本",
+  "intro": "从URL下载PDF文件，并发送至Doc2X进行解析，返回带LaTeX公式的markdown格式的文本",
   "inputExplanationUrl": "https://fael3z0zfze.feishu.cn/wiki/Rkc5witXWiJoi5kORd2cofh6nDg?fromScene=spaceOverview",
   "showStatus": true,
   "weight": 10,
@@ -201,7 +201,7 @@
             "key": "system_httpJsonBody",
             "renderTypeList": ["hidden"],
             "valueType": "any",
-            "value": "{\n  \"apikey\": \"{{apikey}}\",\n  \"url\": \"{{url}}\",\n  \"ocr\": \"{{ocr}}\"\n}",
+            "value": "{\n  \"apikey\": \"{{apikey}}\",\n  \"url\": \"{{url}}\",\n  \"ocr\": {{ocr}}\n}",
             "label": "",
             "required": false
           },

packages/plugins/src/Doc2X/template.json

@@ -3,7 +3,7 @@
   "version": "488",
   "name": "Doc2X服务",
   "avatar": "plugins/doc2x",
-  "intro": "传入的URL形式的图片或PDF文件发送至Doc2X进行解析，返回带LaTeX公式的markdown格式的文本。",
+  "intro": "将传入的图片或PDF文件发送至Doc2X进行解析，返回带LaTeX公式的markdown格式的文本。",
   "showStatus": true,
   "weight": 10,