perf: url fetch

client/package.json

@@ -59,7 +59,7 @@
     "request-ip": "^3.3.0",
     "request-ip": "^3.3.0",
     "sass": "^1.58.3",
     "sass": "^1.58.3",
     "tunnel": "^0.0.6",
     "tunnel": "^0.0.6",
-    "jsdom": "^22.1.0"
+    "jsdom": "^22.1.0",
     "winston": "^3.10.0",
     "winston": "^3.10.0",
     "winston-mongodb": "^5.1.1",
     "winston-mongodb": "^5.1.1",
     "zustand": "^4.3.5"
     "zustand": "^4.3.5"

client/src/pages/api/plugins/kb/fetchContent.ts

-// pages/api/fetchContent.ts
-import { NextApiRequest, NextApiResponse } from 'next';
-import axios from 'axios';
-import { JSDOM } from 'jsdom';
-import { Readability } from '@mozilla/readability';
-import { jsonRes } from '@/service/response';
-
-const fetchContent = async (req: NextApiRequest, res: NextApiResponse) => {
-  const { url } = req.body;
-  if (!url) {
-    return res.status(400).json({ error: 'URL is required' });
-  }
-
-  try {
-    const response = await axios.get(url, {
-      httpsAgent: new (require('https').Agent)({
-        rejectUnauthorized: false,
-      }),
-    });
-
-    const dom = new JSDOM(response.data, {
-      url,
-      contentType: 'text/html',
-    });
-
-    const reader = new Readability(dom.window.document);
-    const article = reader.parse();
-
-    if (!article) {
-        jsonRes(res, {
-            code: 500,
-            error: '页面获取失败或页面为空'
-          });
-          return;
-    }
-
-    jsonRes(res, {
-        code: 200,
-        data: article.content
-      });
-    
-  } catch (error:any) {
-    jsonRes(res, {
-        code: 500,
-        error: error
-      });
-    }
-  
-};
-
-export default fetchContent;

client/src/pages/api/plugins/urlFetch.ts

+// pages/api/fetchContent.ts
+import { NextApiRequest, NextApiResponse } from 'next';
+import axios from 'axios';
+import { JSDOM } from 'jsdom';
+import { Readability } from '@mozilla/readability';
+import { jsonRes } from '@/service/response';
+import { authUser } from '@/service/utils/auth';
+
+type FetchResultItem = {
+  url: string;
+  title: string;
+  content: string;
+};
+export type UrlFetchResponse = FetchResultItem[];
+
+const fetchContent = async (req: NextApiRequest, res: NextApiResponse) => {
+  try {
+    const { urlList = [] } = req.body as { urlList: string[] };
+
+    if (!urlList || urlList.length === 0) {
+      throw new Error('urlList is empty');
+    }
+
+    await authUser({ req });
+
+    const response = (
+      await Promise.allSettled(
+        urlList.map(async (url) => {
+          const fetchRes = await axios.get(url, {
+            timeout: 30000
+          });
+
+          const dom = new JSDOM(fetchRes.data, {
+            url,
+            contentType: 'text/html'
+          });
+
+          const reader = new Readability(dom.window.document);
+          const article = reader.parse();
+
+          return {
+            url,
+            title: article?.title || '',
+            content: article?.textContent || ''
+          };
+        })
+      )
+    )
+      .filter((item) => item.status === 'fulfilled')
+      .map((item: any) => item.value)
+      .filter((item) => item.content);
+
+    jsonRes<UrlFetchResponse>(res, {
+      data: response
+    });
+  } catch (error: any) {
+    jsonRes(res, {
+      code: 500,
+      error: error
+    });
+  }
+};
+
+export default fetchContent;