add env proxypool (#3939)

plugins/webcrawler/SPIDER/.env.example

@@ -18,4 +18,6 @@ MONGODB_URI=mongodb://root:example@mongodb:27017
 BLACKLIST = [".gov.cn",".edu.cn"]
 
 STD_TTL=3600
-EXPIRE_AFTER_SECONDS=9000
\ No newline at end of file
+EXPIRE_AFTER_SECONDS=9000
+
+#VALIDATE_PROXY=[{"ip":"","port":},{"ip":"","port":}]
\ No newline at end of file

plugins/webcrawler/SPIDER/src/utils/setupPage.ts

 import { Page } from 'puppeteer';
 import randomUseragent from 'random-useragent';
+import dotenv from 'dotenv';
 
+dotenv.config();
 const getRandomUserAgent = () => {
   return randomUseragent.getRandom();
 };
@@ -11,31 +13,10 @@ const getRandomPlatform = () => {
 };
 
 //代理池
-const validateproxy = [
-  { ip: "39.102.210.222", port: 8080 },
-  { ip: "8.130.71.75", port: 8080 },
-  { ip: "39.102.214.208", port: 9999 },
-  { ip: "39.104.59.56", port: 8080 },
-  { ip: "8.130.37.235", port: 3128 },
-  { ip: "8.138.131.110", port: 8080 },
-  { ip: "8.140.105.75", port: 8009 },
-  { ip: "114.80.38.120", port: 3081 },
-  { ip: "8.148.23.165", port: 8081 },
-  { ip: "119.96.72.199", port: 59394 },
-  { ip: "120.55.14.137", port: 80 },
-  { ip: "47.116.181.146", port: 5060 },
-  { ip: "39.102.214.199", port: 3128 },
-  { ip: "47.121.183.107", port: 8080 },
-  { ip: "39.104.16.201", port: 8080 },
-  { ip: "39.102.209.163", port: 10002 },
-  { ip: "101.201.76.157", port: 9090 },
-  { ip: "122.224.124.26", port: 12080 },
-  { ip: "180.105.244.199", port: 1080 },
-  { ip: "119.3.113.150", port: 9094 }
-];
+const validateproxy = process.env.VALIDATE_PROXY ? JSON.parse(process.env.VALIDATE_PROXY) : [];
 
 const getRandomProxy = () => {
-  return validateproxy[Math.floor(Math.random() * validateproxy.length)];
+  return validateproxy.length > 0 ? validateproxy[Math.floor(Math.random() * validateproxy.length)] : null;
 };
 
 const getRandomLanguages = () => {
@@ -49,10 +30,12 @@ const getRandomLanguages = () => {
 
 export const setupPage = async (page: Page): Promise<void> => {
   const proxy = getRandomProxy();
-  await page.authenticate({
-    username: proxy.ip,
-    password: proxy.port.toString()
-  });
+  if (proxy) {
+    await page.authenticate({
+      username: proxy.ip,
+      password: proxy.port.toString()
+    });
+  }
 
   await page.evaluateOnNewDocument(() => {
     const newProto = (navigator as any).__proto__;