feat: Text check before synchronization (#689)

* fix: icon

* fix: web selector

* fix: web selector

* perf: link sync

* dev doc

* chomd doc

* perf: git intro

* 466 intro

* intro img

* add json editor (#5)

* team limit

* websync limit

* json editor

* text editor

* perf: search test

* change cq value type

* doc

* intro img

---------

Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>

docSite/content/docs/development/intro.md

@@ -71,6 +71,8 @@ git clone git@github.com:<github_username>/FastGPT.git
 ### 5. 运行
 
 ```bash
+# 给脚本代码执行权限
+chmod -R +x ./scripts/
 # 代码根目录下执行，会安装根 package、projects 和 packages 内所有依赖
 pnpm i
 # 切换到应用目录
@@ -105,6 +107,10 @@ docker build -t dockername/fastgpt:tag --build-arg name=app --build-arg proxy=ta
 1. 如果你是连接远程的数据库，先检查对应的端口是否开放。
 2. 如果是本地运行的数据库，可尝试`host`改成`localhost`或`127.0.0.1`
 
+### sh ./scripts/postinstall.sh 没权限
+
+FastGPT 在`pnpm i`后会执行`postinstall`脚本，用于自动生成`ChakraUI`的`Type`。如果没有权限，可以先执行`chmod -R +x ./scripts/`，再执行`pnpm i`。
+
 ### 加入社区
 
 遇到困难了吗？有任何问题吗? 加入微信群与开发者和用户保持沟通。

docSite/content/docs/development/qa.md

@@ -31,6 +31,10 @@ OneAPI 中没有配置该模型渠道。
 
 页面中是用 stream=true 模式，所以API也需要设置 stream=true 来进行测试。部分模型接口（国产居多）非 Stream 的兼容有点垃圾。
 
+### Incorrect API key provided: sk-xxxx.You can find your api Key at xxx
+
+OneAPI 的 API Key 配置错误，需要修改`OPENAI_API_KEY`环境变量，并重启容器（先 stop 然后 rm 掉，最后再 up -d 运行一次）。可以`exec`进入容器，`env`查看环境变量是否生效。
+
 ## Docker 部署常见问题
 
 ### 如何更新？
@@ -87,3 +91,15 @@ PG 数据库没有连接上/初始化失败，可以查看日志。FastGPT 会
 mongo连接失败，检查
 1. mongo 服务有没有起来(有些 cpu 不支持 AVX，无法用 mongo5，需要换成 mongo4.x，可以dockerhub找个最新的4.x，修改镜像版本，重新运行）
 2. 环境变量（账号密码，注意host和port）
+
+## 本地开发问题
+
+### TypeError: Cannot read properties of null (reading 'useMemo' )
+
+用 Node18 试试，可能最新的 Node 有问题。 本地开发流程：
+
+1. 根目录: `pnpm i`
+2. 复制 `config.json` -> `config.local.json`
+3. 复制 `.env.template` -> `.env.local`
+4. `cd projects/app`
+5. `pnpm dev`
\ No newline at end of file

docSite/content/docs/development/upgrading/466.md

@@ -13,8 +13,11 @@ weight: 830
 
 ## V4.6.6 更新说明
 
-1. 新增 - 搜索方式：分离向量语义检索，全文检索和重排，通过 RRF 进行排序合并。
-2. 优化 - 问题分类提示词，id引导。测试国产商用 api 模型（百度阿里智谱讯飞）使用 Prompt 模式均可分类。
-3. UI 优化，未来将逐步替换新的UI设计。
-4. 优化代码：Icon 抽离和自动化获取。
-5. 查看 [FastGPT 2024 RoadMap](https://github.com/labring/FastGPT?tab=readme-ov-file#-%E5%9C%A8%E7%BA%BF%E4%BD%BF%E7%94%A8)
\ No newline at end of file
+1. 查看 [FastGPT 2024 RoadMap](https://github.com/labring/FastGPT?tab=readme-ov-file#-%E5%9C%A8%E7%BA%BF%E4%BD%BF%E7%94%A8)
+2. 新增 - Http 模块请求头支持 Json 编辑器。
+3. 新增 - [ReRank模型部署](/docs/development/custom-models/reranker/)
+4. 新增 - 搜索方式：分离向量语义检索，全文检索和重排，通过 RRF 进行排序合并。
+5. 优化 - 问题分类提示词，id引导。测试国产商用 api 模型（百度阿里智谱讯飞）使用 Prompt 模式均可分类。
+6. UI 优化，未来将逐步替换新的UI设计。
+7. 优化代码：Icon 抽离和自动化获取。
+8. 修复 - 链接读取的数据集，未保存选择器，导致同步时不使用选择器。
\ No newline at end of file

packages/global/common/file/api.d.ts

@@ -12,4 +12,5 @@ export type UrlFetchParams = {
 export type UrlFetchResponse = {
   url: string;
   content: string;
+  selector?: string;
 }[];

packages/global/common/string/tiktoken/index.ts

@@ -35,7 +35,7 @@ export function countPromptTokens(
   const text = `${role}\n${prompt}`;
   try {
     const encodeText = enc.encode(text);
-    return encodeText.length + 3; // 补充 role 估算值
+    return encodeText.length + role.length; // 补充 role 估算值
   } catch (error) {
     return text.length;
   }

packages/global/common/system/types/index.d.ts

@@ -44,7 +44,8 @@ export type FastGPTFeConfigsType = {
     google?: string;
   };
   limit?: {
-    exportLimitMinutes?: number;
+    exportDatasetLimitMinutes?: number;
+    websiteSyncLimitMinuted?: number;
   };
   scripts?: { [key: string]: string }[];
   favicon?: string;

packages/global/core/dataset/constant.ts

@@ -73,6 +73,19 @@ export const DatasetCollectionTrainingTypeMap = {
   }
 };
 
+export enum DatasetCollectionSyncResultEnum {
+  sameRaw = 'sameRaw',
+  success = 'success'
+}
+export const DatasetCollectionSyncResultMap = {
+  [DatasetCollectionSyncResultEnum.sameRaw]: {
+    label: 'core.dataset.collection.sync.result.sameRaw'
+  },
+  [DatasetCollectionSyncResultEnum.success]: {
+    label: 'core.dataset.collection.sync.result.success'
+  }
+};
+
 /* ------------ data -------------- */
 export enum DatasetDataIndexTypeEnum {
   chunk = 'chunk',
@@ -150,16 +163,24 @@ export enum SearchScoreTypeEnum {
 }
 export const SearchScoreTypeMap = {
   [SearchScoreTypeEnum.embedding]: {
-    label: 'core.dataset.search.score.embedding'
+    label: 'core.dataset.search.score.embedding',
+    desc: 'core.dataset.search.score.embedding desc',
+    showScore: true
   },
   [SearchScoreTypeEnum.fullText]: {
-    label: 'core.dataset.search.score.fullText'
+    label: 'core.dataset.search.score.fullText',
+    desc: 'core.dataset.search.score.fullText desc',
+    showScore: false
   },
   [SearchScoreTypeEnum.reRank]: {
-    label: 'core.dataset.search.score.reRank'
+    label: 'core.dataset.search.score.reRank',
+    desc: 'core.dataset.search.score.reRank desc',
+    showScore: true
   },
   [SearchScoreTypeEnum.rrf]: {
-    label: 'core.dataset.search.score.rrf'
+    label: 'core.dataset.search.score.rrf',
+    desc: 'core.dataset.search.score.rrf desc',
+    showScore: false
   }
 };
 

packages/global/core/dataset/type.d.ts

@@ -49,7 +49,10 @@ export type DatasetCollectionSchemaType = {
   qaPrompt?: string;
   rawTextLength?: number;
   hashRawText?: string;
-  metadata?: Record<string, any>;
+  metadata?: {
+    webPageSelector?: string;
+    [key: string]: any;
+  };
 };
 
 export type DatasetDataIndexItemType = {

packages/global/core/module/node/constant.ts

@@ -7,7 +7,10 @@ export enum FlowNodeInputTypeEnum {
   slider = 'slider',
   target = 'target', // data input
   switch = 'switch',
+
+  // editor
   textarea = 'textarea',
+  JSONEditor = 'JSONEditor',
 
   addInputParam = 'addInputParam', // params input
 

packages/global/core/module/template/system/http.ts

@@ -55,7 +55,7 @@ export const HttpModule: FlowModuleTemplateType = {
     },
     {
       key: ModuleInputKeyEnum.httpHeader,
-      type: FlowNodeInputTypeEnum.textarea,
+      type: FlowNodeInputTypeEnum.JSONEditor,
       valueType: ModuleIOValueTypeEnum.string,
       label: 'core.module.input.label.Http Request Header',
       description: 'core.module.input.description.Http Request Header',

packages/global/support/user/team/type.d.ts

@@ -10,6 +10,10 @@ export type TeamSchema = {
   balance: number;
   maxSize: number;
   lastDatasetBillTime: Date;
+  limit: {
+    lastExportDatasetTime: Date;
+    lastWebsiteSyncTime: Date;
+  };
 };
 
 export type TeamMemberSchema = {

packages/global/support/user/type.d.ts

@@ -17,10 +17,6 @@ export type UserModelSchema = {
     key: string;
     baseUrl: string;
   };
-  limit: {
-    exportKbTime?: Date;
-    datasetMaxCount?: number;
-  };
 };
 
 export type UserType = {

packages/service/common/string/cheerio.ts

@@ -15,7 +15,8 @@ export const cheerioToHtml = ({
   // get origin url
   const originUrl = new URL(fetchUrl).origin;
 
-  const selectDom = $(selector || 'body');
+  const usedSelector = selector || 'body';
+  const selectDom = $(usedSelector);
 
   // remove i element
   selectDom.find('i,script').remove();
@@ -49,7 +50,10 @@ export const cheerioToHtml = ({
     .get()
     .join('\n');
 
-  return html;
+  return {
+    html,
+    usedSelector
+  };
 };
 export const urlsFetch = async ({
   urlList,
@@ -66,25 +70,25 @@ export const urlsFetch = async ({
           });
 
           const $ = cheerio.load(fetchRes.data);
-
-          const md = await htmlToMarkdown(
-            cheerioToHtml({
-              fetchUrl: url,
-              $,
-              selector
-            })
-          );
+          const { html, usedSelector } = cheerioToHtml({
+            fetchUrl: url,
+            $,
+            selector
+          });
+          const md = await htmlToMarkdown(html);
 
           return {
             url,
-            content: md
+            content: md,
+            selector: usedSelector
           };
         } catch (error) {
           console.log(error, 'fetch error');
 
           return {
             url,
-            content: ''
+            content: '',
+            selector: ''
           };
         }
       })

packages/service/common/string/markdown.ts

@@ -21,6 +21,9 @@ export const htmlToMarkdown = (html?: string | null) =>
       worker.terminate();
       reject(err);
     });
+    worker.on('exit', (code) => {
+      console.log('html 2 md finish', code);
+    });
 
     worker.postMessage(html);
   });

packages/service/core/dataset/collection/controller.ts

@@ -19,14 +19,16 @@ export async function createOneCollection({
   qaPrompt,
   hashRawText,
   rawTextLength,
-  metadata = {}
-}: CreateDatasetCollectionParams & { teamId: string; tmbId: string }) {
+  metadata = {},
+  ...props
+}: CreateDatasetCollectionParams & { teamId: string; tmbId: string; [key: string]: any }) {
   const { _id } = await MongoDatasetCollection.create({
-    name,
+    ...props,
     teamId,
     tmbId,
-    datasetId,
     parentId: parentId || null,
+    datasetId,
+    name,
     type,
     trainingType,
     chunkSize,

packages/service/core/dataset/collection/schema.ts

@@ -75,6 +75,7 @@ const DatasetCollectionSchema = new Schema({
   qaPrompt: {
     type: String
   },
+
   rawTextLength: {
     type: Number
   },

packages/service/core/dataset/collection/utils.ts

 import type { CollectionWithDatasetType } from '@fastgpt/global/core/dataset/type.d';
 import { MongoDatasetCollection } from './schema';
 import type { ParentTreePathItemType } from '@fastgpt/global/common/parentFolder/type.d';
-import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
 import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter';
 import { MongoDatasetTraining } from '../training/schema';
 import { urlsFetch } from '../../../common/string/cheerio';
 import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
+import { hashStr } from '@fastgpt/global/common/string/tools';
 
 /**
  * get all collection by top collectionId
@@ -65,64 +65,114 @@ export function getCollectionUpdateTime({ name, time }: { time?: Date; name: str
   return new Date();
 }
 
-/* link collection start load data */
-export const loadingOneChunkCollection = async ({
+/**
+ * Get collection raw text by Collection or collectionId
+ */
+export const getCollectionAndRawText = async ({
   collectionId,
-  tmbId,
-  billId,
-  rawText
+  collection,
+  newRawText
 }: {
-  collectionId: string;
-  tmbId: string;
-  billId?: string;
-  rawText?: string;
+  collectionId?: string;
+  collection?: CollectionWithDatasetType;
+  newRawText?: string;
 }) => {
-  const collection = (await MongoDatasetCollection.findById(collectionId).populate(
-    'datasetId'
-  )) as CollectionWithDatasetType;
+  const col = await (async () => {
+    if (collection) return collection;
+    if (collectionId) {
+      return (await MongoDatasetCollection.findById(collectionId).populate(
+        'datasetId'
+      )) as CollectionWithDatasetType;
+    }
 
-  if (!collection) {
-    return Promise.reject(DatasetErrEnum.unCreateCollection);
+    return null;
+  })();
+
+  if (!col) {
+    return Promise.reject('Collection not found');
   }
 
-  const newRawText = await (async () => {
-    if (rawText) return rawText;
+  const rawText = await (async () => {
+    if (newRawText) return newRawText;
     // link
-    if (collection.type === DatasetCollectionTypeEnum.link && collection.rawLink) {
+    if (col.type === DatasetCollectionTypeEnum.link && col.rawLink) {
       // crawl new data
       const result = await urlsFetch({
-        urlList: [collection.rawLink],
-        selector: collection.datasetId?.websiteConfig?.selector
+        urlList: [col.rawLink],
+        selector: col.datasetId?.websiteConfig?.selector || col?.metadata?.webPageSelector
       });
 
       return result[0].content;
     }
+
     // file
 
     return '';
   })();
 
+  const hashRawText = hashStr(rawText);
+  const isSameRawText = col.hashRawText === hashRawText;
+
+  return {
+    collection: col,
+    rawText,
+    isSameRawText
+  };
+};
+
+/* link collection start load data */
+export const reloadCollectionChunks = async ({
+  collectionId,
+  collection,
+  tmbId,
+  billId,
+  rawText
+}: {
+  collectionId?: string;
+  collection?: CollectionWithDatasetType;
+  tmbId: string;
+  billId?: string;
+  rawText?: string;
+}) => {
+  const {
+    rawText: newRawText,
+    collection: col,
+    isSameRawText
+  } = await getCollectionAndRawText({
+    collection,
+    collectionId,
+    newRawText: rawText
+  });
+
+  if (isSameRawText) return;
+
   // split data
   const { chunks } = splitText2Chunks({
     text: newRawText,
-    chunkLen: collection.chunkSize || 512,
+    chunkLen: col.chunkSize || 512,
     countTokens: false
   });
 
   // insert to training queue
   await MongoDatasetTraining.insertMany(
     chunks.map((item, i) => ({
-      teamId: collection.teamId,
+      teamId: col.teamId,
       tmbId,
-      datasetId: collection.datasetId._id,
-      collectionId: collection._id,
+      datasetId: col.datasetId._id,
+      collectionId: col._id,
       billId,
-      mode: collection.trainingType,
+      mode: col.trainingType,
       prompt: '',
-      model: collection.datasetId.vectorModel,
+      model: col.datasetId.vectorModel,
       q: item,
       a: '',
       chunkIndex: i
     }))
   );
+
+  // update raw text
+  await MongoDatasetCollection.findByIdAndUpdate(col._id, {
+    rawTextLength: newRawText.length,
+    hashRawText: hashStr(newRawText)
+  });
 };

packages/service/support/user/schema.ts

@@ -47,15 +47,6 @@ const UserSchema = new Schema({
     type: Number,
     default: 15
   },
-  limit: {
-    exportKbTime: {
-      // Every half hour
-      type: Date
-    },
-    datasetMaxCount: {
-      type: Number
-    }
-  },
   openaiAccount: {
     type: {
       key: String,

packages/service/support/user/team/teamSchema.ts

@@ -32,6 +32,14 @@ const TeamSchema = new Schema({
   },
   lastDatasetBillTime: {
     type: Date
+  },
+  limit: {
+    lastExportDatasetTime: {
+      type: Date
+    },
+    lastWebsiteSyncTime: {
+      type: Date
+    }
   }
 });
 

packages/service/support/user/utils.ts

+import { MongoTeam } from './team/teamSchema';
+
+/* export dataset limit */
+export const updateExportDatasetLimit = async (teamId: string) => {
+  try {
+    await MongoTeam.findByIdAndUpdate(teamId, {
+      'limit.lastExportDatasetTime': new Date()
+    });
+  } catch (error) {}
+};
+export const checkExportDatasetLimit = async ({
+  teamId,
+  limitMinutes = 0
+}: {
+  teamId: string;
+  limitMinutes?: number;
+}) => {
+  const limitMinutesAgo = new Date(Date.now() - limitMinutes * 60 * 1000);
+
+  // auth export times
+  const authTimes = await MongoTeam.findOne(
+    {
+      _id: teamId,
+      $or: [
+        { 'limit.lastExportDatasetTime': { $exists: false } },
+        { 'limit.lastExportDatasetTime': { $lte: limitMinutesAgo } }
+      ]
+    },
+    '_id limit'
+  );
+
+  if (!authTimes) {
+    return Promise.reject(`每个团队，每 ${limitMinutes} 分钟仅可导出一次。`);
+  }
+};
+
+/* web sync limit */
+export const updateWebSyncLimit = async (teamId: string) => {
+  try {
+    await MongoTeam.findByIdAndUpdate(teamId, {
+      'limit.lastWebsiteSyncTime': new Date()
+    });
+  } catch (error) {}
+};
+export const checkWebSyncLimit = async ({
+  teamId,
+  limitMinutes = 0
+}: {
+  teamId: string;
+  limitMinutes?: number;
+}) => {
+  const limitMinutesAgo = new Date(Date.now() - limitMinutes * 60 * 1000);
+
+  // auth export times
+  const authTimes = await MongoTeam.findOne(
+    {
+      _id: teamId,
+      $or: [
+        { 'limit.lastWebsiteSyncTime': { $exists: false } },
+        { 'limit.lastWebsiteSyncTime': { $lte: limitMinutesAgo } }
+      ]
+    },
+    '_id limit'
+  );
+
+  if (!authTimes) {
+    return Promise.reject(`每个团队，每 ${limitMinutes} 分钟仅使用一次同步功能。`);
+  }
+};

packages/web/components/common/Icon/constants.ts

@@ -15,6 +15,7 @@ export const iconPaths = {
   'common/confirm/deleteTip': () => import('./icons/common/confirm/deleteTip.svg'),
   'common/courseLight': () => import('./icons/common/courseLight.svg'),
   'common/customTitleLight': () => import('./icons/common/customTitleLight.svg'),
+  'common/editor/resizer': () => import('./icons/common/editor/resizer.svg'),
   'common/file/move': () => import('./icons/common/file/move.svg'),
   'common/fullScreenLight': () => import('./icons/common/fullScreenLight.svg'),
   'common/gitFill': () => import('./icons/common/gitFill.svg'),
@@ -28,10 +29,12 @@ export const iconPaths = {
   'common/navbar/pluginFill': () => import('./icons/common/navbar/pluginFill.svg'),
   'common/navbar/pluginLight': () => import('./icons/common/navbar/pluginLight.svg'),
   'common/overviewLight': () => import('./icons/common/overviewLight.svg'),
+  'common/paramsLight': () => import('./icons/common/paramsLight.svg'),
   'common/playFill': () => import('./icons/common/playFill.svg'),
   'common/playLight': () => import('./icons/common/playLight.svg'),
   'common/questionLight': () => import('./icons/common/questionLight.svg'),
   'common/refreshLight': () => import('./icons/common/refreshLight.svg'),
+  'common/resultLight': () => import('./icons/common/resultLight.svg'),
   'common/retryLight': () => import('./icons/common/retryLight.svg'),
   'common/rightArrowLight': () => import('./icons/common/rightArrowLight.svg'),
   'common/routePushLight': () => import('./icons/common/routePushLight.svg'),

packages/web/components/common/Icon/icons/common/editor/resizer.svg

+<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1704259732773" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="4183" xmlns:xlink="http://www.w3.org/1999/xlink" width="200" height="200"><path d="M319.20128 974.56128L348.16 1003.52l655.36-655.36-28.95872-28.95872-655.36 655.36zM675.84 1003.52l327.68-327.68-28.95872-28.95872-327.68 327.68L675.84 1003.52z" fill="#000000" p-id="4184"></path></svg>
\ No newline at end of file

packages/web/components/common/Icon/icons/common/paramsLight.svg

+<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 20 20" fill="none">
+    <path fill-rule="evenodd" clip-rule="evenodd"
+        d="M4.16663 1.66663C4.62686 1.66663 4.99996 2.03972 4.99996 2.49996V5.83329C4.99996 6.29353 4.62686 6.66663 4.16663 6.66663C3.70639 6.66663 3.33329 6.29353 3.33329 5.83329V2.49996C3.33329 2.03972 3.70639 1.66663 4.16663 1.66663ZM9.99996 1.66663C10.4602 1.66663 10.8333 2.03972 10.8333 2.49996V5.14221C11.8043 5.48541 12.5 6.41144 12.5 7.49996C12.5 8.88067 11.3807 9.99996 9.99996 9.99996C8.61925 9.99996 7.49996 8.88067 7.49996 7.49996C7.49996 6.41144 8.19563 5.48541 9.16663 5.14221V2.49996C9.16663 2.03972 9.53972 1.66663 9.99996 1.66663ZM15.8333 1.66663C16.2935 1.66663 16.6666 2.03972 16.6666 2.49996V7.49996C16.6666 7.9602 16.2935 8.33329 15.8333 8.33329C15.3731 8.33329 15 7.9602 15 7.49996V2.49996C15 2.03972 15.3731 1.66663 15.8333 1.66663ZM9.99996 6.66663C9.53972 6.66663 9.16663 7.03972 9.16663 7.49996C9.16663 7.9602 9.53972 8.33329 9.99996 8.33329C10.4602 8.33329 10.8333 7.9602 10.8333 7.49996C10.8333 7.03972 10.4602 6.66663 9.99996 6.66663ZM4.16663 9.99996C3.70639 9.99996 3.33329 10.3731 3.33329 10.8333C3.33329 11.2935 3.70639 11.6666 4.16663 11.6666C4.62686 11.6666 4.99996 11.2935 4.99996 10.8333C4.99996 10.3731 4.62686 9.99996 4.16663 9.99996ZM1.66663 10.8333C1.66663 9.45258 2.78591 8.33329 4.16663 8.33329C5.54734 8.33329 6.66663 9.45258 6.66663 10.8333C6.66663 11.9218 5.97095 12.8478 4.99996 13.191V17.5C4.99996 17.9602 4.62686 18.3333 4.16663 18.3333C3.70639 18.3333 3.33329 17.9602 3.33329 17.5L3.33329 13.191C2.3623 12.8478 1.66663 11.9218 1.66663 10.8333ZM15.8333 11.6666C15.3731 11.6666 15 12.0397 15 12.5C15 12.9602 15.3731 13.3333 15.8333 13.3333C16.2935 13.3333 16.6666 12.9602 16.6666 12.5C16.6666 12.0397 16.2935 11.6666 15.8333 11.6666ZM13.3333 12.5C13.3333 11.1192 14.4526 9.99996 15.8333 9.99996C17.214 9.99996 18.3333 11.1192 18.3333 12.5C18.3333 13.5885 17.6376 14.5145 16.6666 14.8577V17.5C16.6666 17.9602 16.2935 18.3333 15.8333 18.3333C15.3731 18.3333 15 17.9602 15 17.5V14.8577C14.029 14.5145 13.3333 13.5885 13.3333 12.5ZM9.99996 11.6666C10.4602 11.6666 10.8333 12.0397 10.8333 12.5V17.5C10.8333 17.9602 10.4602 18.3333 9.99996 18.3333C9.53972 18.3333 9.16663 17.9602 9.16663 17.5V12.5C9.16663 12.0397 9.53972 11.6666 9.99996 11.6666Z" />
+</svg>
\ No newline at end of file

packages/web/components/common/Icon/icons/common/resultLight.svg

+<svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 20 20" fill="none">
+    <path fill-rule="evenodd" clip-rule="evenodd"
+        d="M14.4508 3.34193C15.5145 3.34207 16.0569 3.34752 16.4738 3.55992C16.8501 3.75166 17.1561 4.05762 17.3478 4.43395C17.5658 4.86177 17.5658 5.42182 17.5658 6.54193V15.1447C17.5658 16.2648 17.5658 16.8249 17.3478 17.2527C17.1561 17.629 16.8501 17.935 16.4738 18.1268C16.046 18.3447 15.4859 18.3447 14.3658 18.3447H5.7658C4.64569 18.3447 4.08564 18.3447 3.65781 18.1268C3.28149 17.935 2.97553 17.629 2.78378 17.2527C2.5658 16.8249 2.5658 16.2648 2.5658 15.1447V6.54193C2.5658 5.42182 2.5658 4.86177 2.78378 4.43395C2.97553 4.05762 3.28149 3.75166 3.65781 3.55992C4.07464 3.34753 4.61699 3.34207 5.68053 3.34193C5.70859 3.34193 5.73701 3.34193 5.7658 3.34193H7.35174C7.39932 2.69822 7.67081 2.1168 8.08862 1.67526C8.19058 1.56751 8.30125 1.46809 8.4195 1.37813C8.8766 1.0304 9.44704 0.823975 10.0657 0.823975C10.6843 0.823975 11.2548 1.0304 11.7119 1.37813C11.8301 1.46809 11.9408 1.56751 12.0428 1.67526C12.4606 2.1168 12.7321 2.69822 12.7796 3.34193H14.3658C14.3945 3.34193 14.4229 3.34193 14.4508 3.34193ZM14.3658 5.0086H5.7658C5.17824 5.0086 4.83781 5.00989 4.58792 5.03031C4.47782 5.0393 4.41959 5.05004 4.39536 5.05546C4.34762 5.08386 4.30773 5.12376 4.27933 5.1715C4.27391 5.19572 4.26317 5.25395 4.25418 5.36405C4.23376 5.61395 4.23246 5.95438 4.23246 6.54193V15.1447C4.23246 15.7323 4.23376 16.0727 4.25418 16.3226C4.26317 16.4327 4.27391 16.4909 4.27933 16.5152C4.30773 16.5629 4.34762 16.6028 4.39536 16.6312C4.41959 16.6366 4.47782 16.6474 4.58792 16.6564C4.83781 16.6768 5.17824 16.6781 5.7658 16.6781H14.3658C14.9533 16.6781 15.2938 16.6768 15.5437 16.6564C15.6538 16.6474 15.712 16.6366 15.7362 16.6312C15.784 16.6028 15.8239 16.5629 15.8523 16.5152C15.8577 16.491 15.8684 16.4327 15.8774 16.3226C15.8978 16.0727 15.8991 15.7323 15.8991 15.1447V6.54193C15.8991 5.95438 15.8978 5.61395 15.8774 5.36405C15.8684 5.25395 15.8577 5.19572 15.8523 5.1715C15.8239 5.12376 15.784 5.08386 15.7362 5.05546C15.712 5.05004 15.6538 5.0393 15.5437 5.03031C15.2938 5.00989 14.9533 5.0086 14.3658 5.0086ZM15.7501 5.05895L15.7489 5.05857L15.7501 5.05895ZM15.8488 16.5291L15.8492 16.5278L15.8488 16.5291ZM4.38147 16.6277L4.38274 16.6281L4.38147 16.6277ZM9.35453 2.76643C9.19102 2.91579 9.07469 3.11595 9.03053 3.34193H11.1008C11.0567 3.11595 10.9404 2.91579 10.7768 2.76643C10.5893 2.59513 10.3397 2.49064 10.0657 2.49064C9.79167 2.49064 9.54207 2.59513 9.35453 2.76643Z" />
+    <path
+        d="M6.0281 7.50999C6.0281 7.04976 6.4012 6.67666 6.86144 6.67666H13.4319C13.8921 6.67666 14.2652 7.04976 14.2652 7.50999C14.2652 7.97023 13.8921 8.34333 13.4319 8.34333H6.86144C6.4012 8.34333 6.0281 7.97023 6.0281 7.50999Z" />
+    <path
+        d="M6.0281 10.8433C6.0281 10.3831 6.4012 10.01 6.86144 10.01H13.4319C13.8921 10.01 14.2652 10.3831 14.2652 10.8433C14.2652 11.3036 13.8921 11.6767 13.4319 11.6767H6.86144C6.4012 11.6767 6.0281 11.3036 6.0281 10.8433Z" />
+    <path
+        d="M6.0281 14.1767C6.0281 13.7164 6.4012 13.3433 6.86144 13.3433H10.0657C10.5259 13.3433 10.899 13.7164 10.899 14.1767C10.899 14.6369 10.5259 15.01 10.0657 15.01H6.86144C6.4012 15.01 6.0281 14.6369 6.0281 14.1767Z" />
+</svg>
\ No newline at end of file

packages/web/components/common/Icon/icons/common/text/t.svg

-<?xml version="1.0" standalone="no"?>
-<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1698125580538"
-    class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="8233"
-    xmlns:xlink="http://www.w3.org/1999/xlink">
+<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 14 14" fill="none">
     <path
-        d="M288.633235 288.699238l446.731484 0L735.364719 421.267533l-13.261025-0.088004c-7.720845-30.709419-16.270569-52.302203-25.91216-65.759703-9.639544-13.458523-22.638603-24.233937-39.127136-32.477692-9.16166-4.03285-25.345248-6.083555-48.682769-6.083555l-51.123354 0-0.611937 336.608388c-1.394766 23.839965 1.394766 18.059308 0 33.261545-0.611937 6.783497 13.173021 20.022009 21.375843 24.733311 8.28878 4.602831 33.500998 5.692652 47.63388 5.692652l19.107173 0 0 18.735713L378.145921 735.890186 378.145921 715.88762l21.98471 0c14.394848 0 31.495318-1.176802 40.045041-6.194072 6.457062-3.533476 26.959002-1.373277 27.176966-24.25338 0.87288-31.494295 2.7486-8.723685 0-31.974225L467.352639 316.857555l-50.906413 0c-33.414017 0-57.537438 6.957459-72.5432 21.00336-21.331841 19.629059-34.767851 47.197952-40.35101 83.491553l-15.485693 0L288.633235 288.699238zM958.709483 244.030899l0 536.025183c0 98.629321-79.960123 178.608887-178.718381 178.608887L243.964896 958.664969c-98.585319 0-178.674379-79.979566-178.674379-178.608887L65.290517 244.030899c0-98.717326 80.090083-178.696892 178.674379-178.696892l536.026206 0C878.74936 65.334008 958.709483 145.314597 958.709483 244.030899zM869.413737 288.699238c0-74.003458-60.067051-134.071532-134.049019-134.071532L288.633235 154.627706c-73.981968 0-133.919059 60.067051-133.919059 134.071532l0 446.601524c0 74.091462 59.937091 134.049019 133.919059 134.049019l446.731484 0c73.981968 0 134.049019-59.957557 134.049019-134.049019L869.413737 288.699238z"
-        p-id="8234"></path>
+        d="M4.1417 4.31668V5.83334C4.1417 5.96221 4.24617 6.06668 4.37503 6.06668H5.07503C5.2039 6.06668 5.30837 5.96221 5.30837 5.83334V5.25001H6.47503V8.77626H5.80068C5.67182 8.77626 5.56735 8.88072 5.56735 9.00959V9.70959C5.56735 9.83845 5.67182 9.94292 5.80068 9.94292H8.31605C8.44492 9.94292 8.54938 9.83845 8.54938 9.70959V9.00959C8.54938 8.88072 8.44492 8.77626 8.31605 8.77626H7.6417V5.25001H8.80837V5.83334C8.80837 5.96221 8.91283 6.06668 9.0417 6.06668H9.7417C9.87057 6.06668 9.97503 5.96221 9.97503 5.83334V4.31668C9.97503 4.18781 9.87057 4.08334 9.7417 4.08334H4.37503C4.24617 4.08334 4.1417 4.18781 4.1417 4.31668Z" />
+    <path fill-rule="evenodd" clip-rule="evenodd"
+        d="M4.52587 1.16675H9.47405C9.9436 1.16674 10.3312 1.16674 10.6469 1.19253C10.9747 1.21932 11.2762 1.27681 11.5593 1.42107C11.9983 1.64477 12.3553 2.00173 12.579 2.44077C12.7232 2.72388 12.7807 3.02529 12.8075 3.35318C12.8333 3.66888 12.8333 4.05642 12.8333 4.52598V9.47419C12.8333 9.94375 12.8333 10.3313 12.8075 10.647C12.7807 10.9749 12.7232 11.2763 12.579 11.5594C12.3553 11.9984 11.9983 12.3554 11.5593 12.5791C11.2762 12.7234 10.9747 12.7808 10.6469 12.8076C10.3312 12.8334 9.94362 12.8334 9.47407 12.8334H4.52591C4.05633 12.8334 3.66877 12.8334 3.35306 12.8076C3.02517 12.7808 2.72376 12.7233 2.44065 12.5791C2.0016 12.3554 1.64465 11.9984 1.42094 11.5594C1.27669 11.2763 1.2192 10.9749 1.19241 10.647C1.16661 10.3313 1.16662 9.94374 1.16663 9.47417V4.52599C1.16662 4.05643 1.16661 3.66888 1.19241 3.35318C1.2192 3.02529 1.27669 2.72388 1.42094 2.44077C1.64465 2.00173 2.0016 1.64477 2.44065 1.42107C2.72376 1.27681 3.02517 1.21932 3.35306 1.19253C3.66876 1.16673 4.05631 1.16674 4.52587 1.16675ZM3.44807 2.35532C3.19232 2.37622 3.06154 2.41409 2.9703 2.46057C2.75078 2.57243 2.5723 2.7509 2.46045 2.97043C2.41397 3.06166 2.37609 3.19245 2.3552 3.44819C2.33375 3.71074 2.33329 4.05041 2.33329 4.55008V9.45008C2.33329 9.94975 2.33375 10.2894 2.3552 10.552C2.37609 10.8077 2.41397 10.9385 2.46045 11.0297C2.5723 11.2493 2.75078 11.4277 2.9703 11.5396C3.06154 11.5861 3.19232 11.6239 3.44807 11.6448C3.71062 11.6663 4.05029 11.6667 4.54996 11.6667H9.44996C9.94963 11.6667 10.2893 11.6663 10.5519 11.6448C10.8076 11.6239 10.9384 11.5861 11.0296 11.5396C11.2491 11.4277 11.4276 11.2493 11.5395 11.0297C11.586 10.9385 11.6238 10.8077 11.6447 10.552C11.6662 10.2894 11.6666 9.94975 11.6666 9.45008V4.55008C11.6666 4.05041 11.6662 3.71074 11.6447 3.44819C11.6238 3.19245 11.586 3.06166 11.5395 2.97043C11.4276 2.75091 11.2491 2.57243 11.0296 2.46058C10.9384 2.41409 10.8076 2.37622 10.5519 2.35532C10.2893 2.33387 9.94963 2.33342 9.44996 2.33342L4.54996 2.33342C4.05029 2.33342 3.71062 2.33387 3.44807 2.35532Z" />
 </svg>
\ No newline at end of file

packages/web/components/common/Icon/icons/history.svg

-<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1686557412109" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="2150" xmlns:xlink="http://www.w3.org/1999/xlink" ><path d="M511.998 64C264.574 64 64 264.574 64 511.998S264.574 960 511.998 960 960 759.422 960 511.998 759.422 64 511.998 64z m353.851 597.438c-82.215 194.648-306.657 285.794-501.306 203.579S78.749 558.36 160.964 363.711 467.621 77.917 662.27 160.132c168.009 70.963 262.57 250.652 225.926 429.313a383.995 383.995 0 0 1-22.347 71.993z"  p-id="2151"></path><path d="M543.311 498.639V256.121c0-17.657-14.314-31.97-31.97-31.97s-31.97 14.314-31.97 31.97v269.005l201.481 201.481c12.485 12.485 32.728 12.485 45.213 0s12.485-32.728 0-45.213L543.311 498.639z" p-id="2152"></path></svg>
\ No newline at end of file
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20">
+    <path fill-rule="evenodd" clip-rule="evenodd"
+        d="M10 3.08634C6.18178 3.08634 3.08647 6.18166 3.08647 9.99992C3.08647 13.8182 6.18178 16.9135 10 16.9135C13.8183 16.9135 16.9136 13.8182 16.9136 9.99992C16.9136 6.18166 13.8183 3.08634 10 3.08634ZM1.4198 9.99992C1.4198 5.26118 5.26131 1.41968 10 1.41968C14.7388 1.41968 18.5803 5.26118 18.5803 9.99992C18.5803 14.7387 14.7388 18.5802 10 18.5802C5.26131 18.5802 1.4198 14.7387 1.4198 9.99992ZM10 4.51844C10.4603 4.51844 10.8334 4.89154 10.8334 5.35178V9.48489L13.4715 10.8039C13.8831 11.0098 14.05 11.5103 13.8442 11.922C13.6383 12.3336 13.1378 12.5005 12.7261 12.2947L9.62737 10.7453C9.34505 10.6041 9.16671 10.3156 9.16671 9.99992V5.35178C9.16671 4.89154 9.53981 4.51844 10 4.51844Z" />
+</svg>
\ No newline at end of file

packages/web/components/common/Textarea/JsonEditor/index.tsx

+import React from 'react';
+import Editor from '@monaco-editor/react';
+import { useCallback, useRef, useState } from 'react';
+import { Box, BoxProps } from '@chakra-ui/react';
+import MyIcon from '../../Icon';
+
+type Props = Omit<BoxProps, 'onChange' | 'resize' | 'height'> & {
+  height?: number;
+  resize?: boolean;
+  defaultValue?: string;
+  value?: string;
+  onChange?: (e: string) => void;
+};
+
+const options = {
+  lineNumbers: 'off',
+  guides: {
+    indentation: false
+  },
+  automaticLayout: true,
+  minimap: {
+    enabled: false
+  },
+  scrollbar: {
+    verticalScrollbarSize: 4,
+    horizontalScrollbarSize: 8,
+    alwaysConsumeMouseWheel: false
+  },
+  lineNumbersMinChars: 0,
+  fontSize: 12,
+  scrollBeyondLastLine: false,
+  folding: false,
+  overviewRulerBorder: false,
+  tabSize: 2
+};
+
+const JSONEditor = ({ defaultValue, value, onChange, resize, ...props }: Props) => {
+  const [height, setHeight] = useState(props.height || 100);
+  const initialY = useRef(0);
+
+  const handleMouseDown = useCallback((e: React.MouseEvent) => {
+    initialY.current = e.clientY;
+
+    const handleMouseMove = (e: MouseEvent) => {
+      const deltaY = e.clientY - initialY.current;
+      setHeight((prevHeight) => (prevHeight + deltaY < 100 ? 100 : prevHeight + deltaY));
+      initialY.current = e.clientY;
+    };
+
+    const handleMouseUp = () => {
+      document.removeEventListener('mousemove', handleMouseMove);
+      document.removeEventListener('mouseup', handleMouseUp);
+    };
+
+    document.addEventListener('mousemove', handleMouseMove);
+    document.addEventListener('mouseup', handleMouseUp);
+  }, []);
+
+  return (
+    <Box position={'relative'}>
+      {resize && (
+        <Box
+          position={'absolute'}
+          right={'0'}
+          bottom={'0'}
+          zIndex={999}
+          cursor={'ns-resize'}
+          px={'4px'}
+          onMouseDown={handleMouseDown}
+        >
+          <MyIcon name={'common/editor/resizer'} width={'16px'} height={'16px'} />
+        </Box>
+      )}
+
+      <Box
+        borderWidth={'1px'}
+        borderRadius={'base'}
+        borderColor={'myGray.200'}
+        py={2}
+        {...props}
+        height={'auto'}
+      >
+        <Editor
+          height={height}
+          defaultLanguage="json"
+          options={options as any}
+          theme={'JSONEditorTheme'}
+          beforeMount={(monaco) => {
+            monaco?.editor.defineTheme('JSONEditorTheme', {
+              base: 'vs',
+              inherit: true,
+              rules: [],
+              colors: {
+                'editor.background': '#ffffff00',
+                'editorLineNumber.foreground': '#aaa',
+                'editorOverviewRuler.border': '#ffffff00',
+                'editor.lineHighlightBackground': '#F7F8FA',
+                'scrollbarSlider.background': '#E8EAEC',
+                'editorIndentGuide.activeBackground': '#ddd',
+                'editorIndentGuide.background': '#eee'
+              }
+            });
+          }}
+          defaultValue={defaultValue}
+          value={value}
+          onChange={(e) => onChange?.(e || '')}
+        />
+      </Box>
+    </Box>
+  );
+};
+
+export default React.memo(JSONEditor);

packages/web/package.json

@@ -13,6 +13,7 @@
     "@chakra-ui/system": "^2.6.1",
     "@emotion/react": "^11.11.1",
     "@emotion/styled": "^11.11.0",
+    "@monaco-editor/react": "^4.6.0",
     "react": "18.2.0",
     "react-dom": "18.2.0",
     "i18next": "^22.5.1",

pnpm-workspace.yaml

 packages:
   - 'packages/*'
   - 'projects/*'
+  - 'scripts/icon'

projects/app/public/docs/versionIntro.md

 ### Fast GPT V4.6.6
 
-1. 新增 - [问题补全模块](https://doc.fastgpt.in/docs/workflow/modules/coreferenceresolution/)
-2. 新增 - [文本编辑模块](https://doc.fastgpt.in/docs/workflow/modules/text_editor/)
-3. 新增 - [判断器模块](https://doc.fastgpt.in/docs/workflow/modules/tfswitch/)
-4. 新增 - [自定义反馈模块](https://doc.fastgpt.in/docs/workflow/modules/custom_feedback/)
-5. 新增 - 【内容提取】模块支持选择模型，以及字段枚举
-6. 优化 - docx读取，兼容表格（表格转markdown）
-7. 优化 - 高级编排连接线交互
-8. 优化 - 由于 html2md 导致的 cpu密集计算，阻断线程问题
-9. 修复 - 高级编排提示词提取描述
-10. [使用文档](https://doc.fastgpt.in/docs/intro/)
-11. [点击查看高级编排介绍文档](https://doc.fastgpt.in/docs/workflow)
-12. [点击查看商业版](https://doc.fastgpt.in/docs/commercial/)
+1. 新增 - Http 模块请求头支持 Json 编辑器。
+2. 新增 - 搜索方式：分离向量语义检索，全文检索和重排，通过 RRF 进行排序合并。
+3. 新增 - [问题补全模块](https://doc.fastgpt.in/docs/workflow/modules/coreferenceresolution/)
+5. 新增 - [文本编辑模块](https://doc.fastgpt.in/docs/workflow/modules/text_editor/)
+6. 新增 - [判断器模块](https://doc.fastgpt.in/docs/workflow/modules/tfswitch/)
+7. 新增 - [自定义反馈模块](https://doc.fastgpt.in/docs/workflow/modules/custom_feedback/)
+8. 新增 - 【内容提取】模块支持选择模型，以及字段枚举
+9. [使用文档](https://doc.fastgpt.in/docs/intro/)
+10. [点击查看高级编排介绍文档](https://doc.fastgpt.in/docs/workflow)
+11. [点击查看商业版](https://doc.fastgpt.in/docs/commercial/)

projects/app/public/locales/en/common.json

@@ -410,6 +410,7 @@
           "Read Metadata": "Read Metadata",
           "Training Type": "Training Type",
           "Updatetime": "Update Time",
+          "Web page selector": "Web Selector",
           "metadata": "Metadata",
           "read source": "Read Source",
           "source": "Source",
@@ -420,6 +421,12 @@
           "active": "Ready",
           "syncing": "Syncing"
         },
+        "sync": {
+          "result": {
+            "sameRaw": "The content has not changed and no update is required.",
+            "success": "Start synchronization"
+          }
+        },
         "training": {
           "type chunk": "Chunk",
           "type manual": "Manual",

projects/app/public/locales/zh/common.json

@@ -410,6 +410,7 @@
           "Read Metadata": "查看元数据",
           "Training Type": "训练模式",
           "Updatetime": "更新时间",
+          "Web page selector": "网站选择器",
           "metadata": "元数据",
           "read source": "查看原始内容",
           "source": "数据来源",
@@ -420,6 +421,12 @@
           "active": "已就绪",
           "syncing": "同步中"
         },
+        "sync": {
+          "result": {
+            "sameRaw": "内容未变动，无需更新",
+            "success": "开始同步"
+          }
+        },
         "training": {
           "type chunk": "直接分段",
           "type manual": "手动",
@@ -515,9 +522,13 @@
         },
         "score": {
           "embedding": "语义检索",
+          "embedding desc": "通过计算向量之间的距离获取得分，范围为 0~1。",
           "fullText": "全文检索",
+          "fullText desc": "计算相同关键词的得分，范围为 0~无穷。",
           "reRank": "结果重排",
-          "rrf": "RRF 合并"
+          "reRank desc": "通过 ReRank 模型计算句子之间的关联度，范围为 0~1。",
+          "rrf": "综合排名",
+          "rrf desc": "通过倒排计算的方式，合并多个检索结果。"
         },
         "search mode": "搜索模式"
       },

projects/app/src/components/Layout/navbarPhone.tsx

@@ -28,14 +28,14 @@ const NavbarPhone = ({ unread }: { unread: number }) => {
       },
       {
         label: t('navbar.Tools'),
-        icon: 'phoneTabbar/tabbarMore',
+        icon: 'phoneTabbar/more',
         link: '/tools',
         activeLink: ['/tools'],
         unread: 0
       },
       {
         label: t('navbar.Account'),
-        icon: 'phoneTabbar/tabbarMe',
+        icon: 'phoneTabbar/me',
         link: '/account',
         activeLink: ['/account'],
         unread

projects/app/src/components/core/module/Flow/components/nodes/NodeCQNode.tsx

@@ -94,7 +94,7 @@ const NodeCQNode = React.memo(function NodeCQNode({ data }: { data: FlowModuleIt
                         />
                         <SourceHandle
                           handleKey={item.key}
-                          valueType={ModuleIOValueTypeEnum.string}
+                          valueType={ModuleIOValueTypeEnum.boolean}
                         />
                       </Box>
                     </Box>

projects/app/src/components/core/module/Flow/components/render/RenderInput/index.tsx

@@ -64,6 +64,10 @@ const RenderList: {
   {
     types: [FlowNodeInputTypeEnum.addInputParam],
     Component: dynamic(() => import('./templates/AddInputParam'))
+  },
+  {
+    types: [FlowNodeInputTypeEnum.JSONEditor],
+    Component: dynamic(() => import('./templates/JsonEditor'))
   }
 ];
 const UserChatInput = dynamic(() => import('./templates/UserChatInput'));

projects/app/src/components/core/module/Flow/components/render/RenderInput/templates/JsonEditor.tsx

+import React, { useCallback } from 'react';
+import type { RenderInputProps } from '../type';
+import { onChangeNode } from '../../../../FlowProvider';
+import { useTranslation } from 'next-i18next';
+import JSONEditor from '@fastgpt/web/components/common/Textarea/JsonEditor';
+
+const JsonEditor = ({ item, moduleId }: RenderInputProps) => {
+  const { t } = useTranslation();
+
+  const update = useCallback(
+    (value: string) => {
+      onChangeNode({
+        moduleId,
+        type: 'updateInput',
+        key: item.key,
+        value: {
+          ...item,
+          value
+        }
+      });
+    },
+    [item, moduleId]
+  );
+
+  return (
+    <JSONEditor
+      title={t(item.label)}
+      bg={'myWhite.400'}
+      placeholder={t(item.placeholder || '')}
+      resize
+      defaultValue={item.value}
+      onChange={(e) => {
+        update(e);
+      }}
+    />
+  );
+};
+
+export default React.memo(JsonEditor);

projects/app/src/pages/api/common/system/getInitData.ts

@@ -50,7 +50,8 @@ const defaultFeConfigs: FastGPTFeConfigsType = {
   concatMd:
     '* 项目开源地址: [FastGPT GitHub](https://github.com/labring/FastGPT)\n* 交流群: ![](https://doc.fastgpt.in/wechat-fastgpt.webp)',
   limit: {
-    exportLimitMinutes: 0
+    exportDatasetLimitMinutes: 0,
+    websiteSyncLimitMinuted: 0
   },
   scripts: [],
   favicon: '/favicon.ico'

projects/app/src/pages/api/core/dataset/checkExportLimit.ts

-import type { NextApiRequest, NextApiResponse } from 'next';
-import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
-import { MongoUser } from '@fastgpt/service/support/user/schema';
-import { addLog } from '@fastgpt/service/common/system/log';
-import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
-import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
-import { findDatasetIdTreeByTopDatasetId } from '@fastgpt/service/core/dataset/controller';
-
-export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
-  try {
-    await connectToDatabase();
-    let { datasetId } = req.query as {
-      datasetId: string;
-    };
-
-    if (!datasetId) {
-      throw new Error('缺少参数');
-    }
-
-    // 凭证校验
-    const { userId } = await authDataset({ req, authToken: true, datasetId, per: 'w' });
-
-    await limitCheck({
-      datasetId,
-      userId
-    });
-
-    jsonRes(res);
-  } catch (err) {
-    res.status(500);
-    jsonRes(res, {
-      code: 500,
-      error: err
-    });
-  }
-}
-
-export async function limitCheck({ datasetId, userId }: { datasetId: string; userId: string }) {
-  const exportIds = await findDatasetIdTreeByTopDatasetId(datasetId);
-
-  const limitMinutesAgo = new Date(
-    Date.now() - (global.feConfigs?.limit?.exportLimitMinutes || 0) * 60 * 1000
-  );
-
-  // auth export times
-  const authTimes = await MongoUser.findOne(
-    {
-      _id: userId,
-      $or: [
-        { 'limit.exportKbTime': { $exists: false } },
-        { 'limit.exportKbTime': { $lte: limitMinutesAgo } }
-      ]
-    },
-    '_id limit'
-  );
-
-  if (!authTimes) {
-    const minutes = `${global.feConfigs?.limit?.exportLimitMinutes || 0} 分钟`;
-    return Promise.reject(`上次导出未到 ${minutes}，每 ${minutes}仅可导出一次。`);
-  }
-
-  // auth max data
-  const total = await MongoDatasetData.countDocuments({
-    datasetId: { $in: exportIds }
-  });
-
-  addLog.info(`export datasets: ${datasetId}`, { total });
-
-  if (total > 100000) {
-    return Promise.reject('数据量超出 10 万，无法导出');
-  }
-}

projects/app/src/pages/api/core/dataset/collection/sync/link.ts

@@ -2,14 +2,20 @@ import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
 import { connectToDatabase } from '@/service/mongo';
 import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset';
-import { loadingOneChunkCollection } from '@fastgpt/service/core/dataset/collection/utils';
+import {
+  getCollectionAndRawText,
+  reloadCollectionChunks
+} from '@fastgpt/service/core/dataset/collection/utils';
 import { delCollectionRelevantData } from '@fastgpt/service/core/dataset/data/controller';
-import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
-import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
+import {
+  DatasetCollectionSyncResultEnum,
+  DatasetCollectionTypeEnum
+} from '@fastgpt/global/core/dataset/constant';
 import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
 import { createTrainingBill } from '@fastgpt/service/support/wallet/bill/controller';
 import { BillSourceEnum } from '@fastgpt/global/support/wallet/bill/constants';
 import { getQAModel, getVectorModel } from '@/service/core/ai/model';
+import { createOneCollection } from '@fastgpt/service/core/dataset/collection/controller';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
@@ -32,6 +38,18 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
       return Promise.reject(DatasetErrEnum.unLinkCollection);
     }
 
+    const { rawText, isSameRawText } = await getCollectionAndRawText({
+      collection
+    });
+
+    if (isSameRawText) {
+      return jsonRes(res, {
+        data: DatasetCollectionSyncResultEnum.sameRaw
+      });
+    }
+
+    /* Not the same original text, create and reload */
+
     const vectorModelData = getVectorModel(collection.datasetId.vectorModel);
     const agentModelData = getQAModel(collection.datasetId.agentModel);
     // create training bill
@@ -45,26 +63,27 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     });
 
     // create a collection and delete old
-    const { _id } = await MongoDatasetCollection.create({
-      parentId: collection.parentId,
+    const _id = await createOneCollection({
       teamId: collection.teamId,
       tmbId: collection.tmbId,
+      parentId: collection.parentId,
       datasetId: collection.datasetId._id,
-      type: collection.type,
       name: collection.name,
-      createTime: collection.createTime,
+      type: collection.type,
       trainingType: collection.trainingType,
       chunkSize: collection.chunkSize,
       fileId: collection.fileId,
       rawLink: collection.rawLink,
-      metadata: collection.metadata
+      metadata: collection.metadata,
+      createTime: collection.createTime
     });
 
     // start load
-    await loadingOneChunkCollection({
+    await reloadCollectionChunks({
       collectionId: _id,
       tmbId,
-      billId
+      billId,
+      rawText
     });
 
     // delete old collection
@@ -73,7 +92,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
       fileIds: collection.fileId ? [collection.fileId] : []
     });
 
-    jsonRes(res);
+    jsonRes(res, {
+      data: DatasetCollectionSyncResultEnum.success
+    });
   } catch (err) {
     jsonRes(res, {
       code: 500,

projects/app/src/pages/api/core/dataset/exportAll.ts

 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes, responseWriteController } from '@fastgpt/service/common/response';
 import { connectToDatabase } from '@/service/mongo';
-import { MongoUser } from '@fastgpt/service/support/user/schema';
 import { addLog } from '@fastgpt/service/common/system/log';
 import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
 import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
 import { findDatasetIdTreeByTopDatasetId } from '@fastgpt/service/core/dataset/controller';
-import { limitCheck } from './checkExportLimit';
 import { withNextCors } from '@fastgpt/service/common/middle/cors';
+import {
+  checkExportDatasetLimit,
+  updateExportDatasetLimit
+} from '@fastgpt/service/support/user/utils';
 
 export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
@@ -21,11 +23,11 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
     }
 
     // 凭证校验
-    const { userId } = await authDataset({ req, authToken: true, datasetId, per: 'w' });
+    const { teamId } = await authDataset({ req, authToken: true, datasetId, per: 'w' });
 
-    await limitCheck({
-      userId,
-      datasetId
+    await checkExportDatasetLimit({
+      teamId,
+      limitMinutes: global.feConfigs?.limit?.exportDatasetLimitMinutes
     });
 
     const exportIds = await findDatasetIdTreeByTopDatasetId(datasetId);
@@ -43,7 +45,9 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
         datasetId: { $in: exportIds }
       },
       'q a'
-    ).cursor();
+    )
+      .limit(50000)
+      .cursor();
 
     const write = responseWriteController({
       res,
@@ -59,12 +63,10 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
       write(`\n"${q}","${a}"`);
     });
 
-    cursor.on('end', async () => {
+    cursor.on('end', () => {
       cursor.close();
       res.end();
-      await MongoUser.findByIdAndUpdate(userId, {
-        'limit.exportKbTime': new Date()
-      });
+      updateExportDatasetLimit(teamId);
     });
 
     cursor.on('error', (err) => {

projects/app/src/pages/api/support/user/team/limit/exportDatasetLimit.ts

+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@fastgpt/service/common/response';
+import { connectToDatabase } from '@/service/mongo';
+import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
+import { checkExportDatasetLimit } from '@fastgpt/service/support/user/utils';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
+  try {
+    await connectToDatabase();
+    const { datasetId } = req.query as {
+      datasetId: string;
+    };
+
+    if (!datasetId) {
+      throw new Error('datasetId is required');
+    }
+
+    // 凭证校验
+    const { teamId } = await authDataset({ req, authToken: true, datasetId, per: 'w' });
+
+    await checkExportDatasetLimit({
+      teamId,
+      limitMinutes: global.feConfigs?.limit?.exportDatasetLimitMinutes
+    });
+
+    jsonRes(res);
+  } catch (err) {
+    res.status(500);
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}

projects/app/src/pages/api/support/user/team/limit/webSyncLimit.ts

+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@fastgpt/service/common/response';
+import { connectToDatabase } from '@/service/mongo';
+import { checkWebSyncLimit } from '@fastgpt/service/support/user/utils';
+import { authCert } from '@fastgpt/service/support/permission/auth/common';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
+  try {
+    await connectToDatabase();
+
+    // 凭证校验
+    const { teamId } = await authCert({ req, authToken: true });
+
+    await checkWebSyncLimit({
+      teamId,
+      limitMinutes: global.feConfigs?.limit?.websiteSyncLimitMinuted
+    });
+
+    jsonRes(res);
+  } catch (err) {
+    res.status(500);
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}

projects/app/src/pages/dataset/detail/components/CollectionCard.tsx

@@ -46,7 +46,8 @@ import {
   DatasetCollectionTrainingModeEnum,
   DatasetTypeEnum,
   DatasetTypeMap,
-  DatasetStatusEnum
+  DatasetStatusEnum,
+  DatasetCollectionSyncResultMap
 } from '@fastgpt/global/core/dataset/constant';
 import { getCollectionIcon } from '@fastgpt/global/core/dataset/utils';
 import EditFolderModal, { useEditFolder } from '../../component/EditFolderModal';
@@ -61,6 +62,7 @@ import { useUserStore } from '@/web/support/user/useUserStore';
 import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant';
 import { useDatasetStore } from '@/web/core/dataset/store/dataset';
 import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type';
+import { DatasetCollectionSyncResultEnum } from '../../../../../../../packages/global/core/dataset/constant';
 
 const FileImportModal = dynamic(() => import('./Import/ImportModal'), {});
 const WebSiteConfigModal = dynamic(() => import('./Import/WebsiteConfig'), {});
@@ -246,8 +248,12 @@ const CollectionCard = () => {
     mutationFn: (collectionId: string) => {
       return postLinkCollectionSync(collectionId);
     },
-    onSuccess() {
+    onSuccess(res: DatasetCollectionSyncResultEnum) {
       getData(pageNum);
+      toast({
+        status: 'success',
+        title: t(DatasetCollectionSyncResultMap[res]?.label)
+      });
     },
     errorToast: t('core.dataset.error.Start Sync Failed')
   });

projects/app/src/pages/dataset/detail/components/DataCard.tsx

@@ -121,46 +121,55 @@ const DataCard = () => {
     [collection?.canWrite, userInfo?.team?.role]
   );
 
-  const metadataList = useMemo(
-    () =>
-      collection
+  const metadataList = useMemo(() => {
+    if (!collection) return [];
+
+    const webSelector =
+      collection?.datasetId?.websiteConfig?.selector || collection?.metadata?.webPageSelector;
+
+    return [
+      {
+        label: t('core.dataset.collection.metadata.source'),
+        value: t(DatasetCollectionTypeMap[collection.type]?.name)
+      },
+      {
+        label: t('core.dataset.collection.metadata.source name'),
+        value: collection.file?.filename || collection?.rawLink || collection?.name
+      },
+      {
+        label: t('core.dataset.collection.metadata.source size'),
+        value: collection.file ? formatFileSize(collection.file.length) : '-'
+      },
+      {
+        label: t('core.dataset.collection.metadata.Createtime'),
+        value: formatTime2YMDHM(collection.createTime)
+      },
+      {
+        label: t('core.dataset.collection.metadata.Updatetime'),
+        value: formatTime2YMDHM(collection.updateTime)
+      },
+      {
+        label: t('core.dataset.collection.metadata.Raw text length'),
+        value: collection.rawTextLength ?? '-'
+      },
+      {
+        label: t('core.dataset.collection.metadata.Training Type'),
+        value: t(DatasetCollectionTrainingTypeMap[collection.trainingType]?.label)
+      },
+      {
+        label: t('core.dataset.collection.metadata.Chunk Size'),
+        value: collection.chunkSize || '-'
+      },
+      ...(webSelector
         ? [
             {
-              label: t('core.dataset.collection.metadata.source'),
-              value: t(DatasetCollectionTypeMap[collection.type]?.name)
-            },
-            {
-              label: t('core.dataset.collection.metadata.source name'),
-              value: collection.file?.filename || collection?.rawLink || collection?.name
-            },
-            {
-              label: t('core.dataset.collection.metadata.source size'),
-              value: collection.file ? formatFileSize(collection.file.length) : '-'
-            },
-            {
-              label: t('core.dataset.collection.metadata.Createtime'),
-              value: formatTime2YMDHM(collection.createTime)
-            },
-            {
-              label: t('core.dataset.collection.metadata.Updatetime'),
-              value: formatTime2YMDHM(collection.updateTime)
-            },
-            {
-              label: t('core.dataset.collection.metadata.Raw text length'),
-              value: collection.rawTextLength ?? '-'
-            },
-            {
-              label: t('core.dataset.collection.metadata.Training Type'),
-              value: t(DatasetCollectionTrainingTypeMap[collection.trainingType]?.label)
-            },
-            {
-              label: t('core.dataset.collection.metadata.Chunk Size'),
-              value: collection.chunkSize || '-'
+              label: t('core.dataset.collection.metadata.Web page selector'),
+              value: webSelector
             }
           ]
-        : [],
-    [collection, t]
-  );
+        : [])
+    ];
+  }, [collection, t]);
 
   return (
     <Box ref={BoxRef} position={'relative'} px={5} py={[1, 5]} h={'100%'} overflow={'overlay'}>

projects/app/src/pages/dataset/detail/components/Import/FileSelect.tsx

@@ -41,6 +41,7 @@ export type FileItemType = {
   type: DatasetCollectionTypeEnum.file | DatasetCollectionTypeEnum.link;
   fileId?: string;
   rawLink?: string;
+  metadata?: Record<string, any>;
 };
 
 export interface Props extends BoxProps {
@@ -232,7 +233,7 @@ const FileSelect = ({
   // link fetch
   const onUrlFetch = useCallback(
     (e: UrlFetchResponse) => {
-      const result: FileItemType[] = e.map<FileItemType>(({ url, content }) => {
+      const result: FileItemType[] = e.map<FileItemType>(({ url, content, selector }) => {
         const { chunks, tokens } = splitText2Chunks({
           text: content,
           chunkLen,
@@ -250,7 +251,10 @@ const FileSelect = ({
           chunks: chunks.map((chunk) => ({
             q: chunk,
             a: ''
-          }))
+          })),
+          metadata: {
+            webPageSelector: selector
+          }
         };
       });
       onPushFiles(result);

projects/app/src/pages/dataset/detail/components/Import/Provider.tsx

@@ -156,19 +156,24 @@ const Provider = ({
     return formatModelPrice2Read(totalTokens * inputPrice);
   }, [inputPrice, mode, outputPrice, totalTokens]);
 
-  /* start upload data */
+  /* 
+    start upload data 
+    1. create training bill
+    2. create collection
+    3. upload chunks
+  */
   const { mutate: onclickUpload, isLoading: uploading } = useRequest({
     mutationFn: async (props?: { prompt?: string }) => {
       const { prompt } = props || {};
       let totalInsertion = 0;
       for await (const file of files) {
-        const chunks = file.chunks;
         // create training bill
         const billId = await postCreateTrainingBill({
           name: t('dataset.collections.Create Training Data', { filename: file.filename }),
           vectorModel,
           agentModel
         });
+
         // create a file collection and training bill
         const collectionId = await postDatasetCollection({
           datasetId,
@@ -181,10 +186,12 @@ const Provider = ({
           trainingType: collectionTrainingType,
           qaPrompt: mode === TrainingModeEnum.qa ? prompt : '',
           rawTextLength: file.rawText.length,
-          hashRawText: hashStr(file.rawText)
+          hashRawText: hashStr(file.rawText),
+          metadata: file.metadata
         });
 
-        // upload data
+        // upload chunks
+        const chunks = file.chunks;
         const { insertLen } = await chunksUpload({
           collectionId,
           billId,

projects/app/src/pages/dataset/detail/components/Test.tsx

@@ -60,6 +60,7 @@ const Test = ({ datasetId }: { datasetId: string }) => {
   const [inputType, setInputType] = useState<'text' | 'file'>('text');
   const [datasetTestItem, setDatasetTestItem] = useState<SearchTestStoreItemType>();
   const [refresh, setRefresh] = useState(false);
+  const [isFocus, setIsFocus] = useState(false);
   const { File, onOpen } = useSelectFile({
     fileType: '.csv',
     multiple: false
@@ -169,7 +170,20 @@ const Test = ({ datasetId }: { datasetId: string }) => {
         py={4}
         borderRight={['none', theme.borders.base]}
       >
-        <Box border={'2px solid'} borderColor={'primary.500'} p={3} mx={4} borderRadius={'md'}>
+        <Box
+          border={'2px solid'}
+          p={3}
+          mx={4}
+          borderRadius={'md'}
+          {...(isFocus
+            ? {
+                borderColor: 'primary.500',
+                boxShadow: '0px 0px 0px 2.4px rgba(51, 112, 255, 0.15)'
+              }
+            : {
+                borderColor: 'primary.300'
+              })}
+        >
           {/* header */}
           <Flex alignItems={'center'} justifyContent={'space-between'}>
             <MySelect
@@ -221,8 +235,12 @@ const Test = ({ datasetId }: { datasetId: string }) => {
                 variant={'unstyled'}
                 maxLength={datasetDetail.vectorModel.maxToken}
                 placeholder={t('core.dataset.test.Test Text Placeholder')}
+                onFocus={() => setIsFocus(true)}
                 {...register('inputText', {
-                  required: true
+                  required: true,
+                  onBlur: () => {
+                    setIsFocus(false);
+                  }
                 })}
               />
             )}
@@ -340,25 +358,26 @@ const TestHistories = React.memo(function TestHistories({
   );
   return (
     <>
-      <Flex alignItems={'center'} color={'myGray.600'}>
-        <MyIcon mr={2} name={'history'} w={'16px'} h={'16px'} />
-        <Box fontSize={'2xl'}>{t('core.dataset.test.test history')}</Box>
+      <Flex alignItems={'center'} color={'myGray.900'}>
+        <MyIcon mr={2} name={'history'} w={'18px'} h={'18px'} color={'myGray.900'} />
+        <Box fontSize={'xl'}>{t('core.dataset.test.test history')}</Box>
       </Flex>
       <Box mt={2}>
-        <Flex py={2} fontWeight={'bold'} borderBottom={theme.borders.sm}>
-          <Box flex={'0 0 80px'}>{t('core.dataset.search.search mode')}</Box>
-          <Box flex={1}>{t('core.dataset.test.Test Text')}</Box>
-          <Box flex={'0 0 70px'}>{t('common.Time')}</Box>
-          <Box w={'14px'}></Box>
-        </Flex>
         {testHistories.map((item) => (
           <Flex
             key={item.id}
-            p={1}
+            py={2}
+            px={3}
             alignItems={'center'}
-            borderBottom={theme.borders.base}
+            borderColor={'borderColor.low'}
+            borderWidth={'1px'}
+            borderRadius={'md'}
+            _notLast={{
+              mb: 2
+            }}
             _hover={{
-              bg: '#f4f4f4',
+              borderColor: 'primary.300',
+              boxShadow: '1',
               '& .delete': {
                 display: 'block'
               }
@@ -369,7 +388,7 @@ const TestHistories = React.memo(function TestHistories({
           >
             <Box flex={'0 0 80px'}>
               {DatasetSearchModeMap[item.searchMode] ? (
-                <Flex alignItems={'center'}>
+                <Flex alignItems={'center'} fontWeight={'500'} color={'myGray.500'}>
                   <MyIcon
                     name={DatasetSearchModeMap[item.searchMode].icon as any}
                     w={'12px'}
@@ -381,7 +400,7 @@ const TestHistories = React.memo(function TestHistories({
                 '-'
               )}
             </Box>
-            <Box flex={1} mr={2} wordBreak={'break-all'}>
+            <Box flex={1} mr={2} wordBreak={'break-all'} fontWeight={'400'}>
               {item.text}
             </Box>
             <Box flex={'0 0 70px'}>{formatTimeToChatTime(item.time)}</Box>
@@ -433,13 +452,20 @@ const TestResults = React.memo(function TestResults({
         </Flex>
       ) : (
         <>
-          <Box fontSize={'xl'} color={'myGray.600'}>
+          <Flex fontSize={'xl'} color={'myGray.900'} alignItems={'center'}>
+            <MyIcon name={'common/paramsLight'} w={'18px'} mr={2} />
             {t('core.dataset.test.Test params')}
-          </Box>
-          <TableContainer mb={3} bg={'myGray.150'} borderRadius={'md'}>
+          </Flex>
+          <TableContainer
+            mt={3}
+            bg={'primary.50'}
+            borderRadius={'lg'}
+            borderWidth={'1px'}
+            borderColor={'primary.1'}
+          >
             <Table>
               <Thead>
-                <Tr>
+                <Tr color={'myGray.600'}>
                   <Th>{t('core.dataset.search.search mode')}</Th>
                   <Th>{t('core.dataset.search.ReRank')}</Th>
                   <Th>{t('core.dataset.search.Max Tokens')}</Th>
@@ -447,8 +473,8 @@ const TestResults = React.memo(function TestResults({
                 </Tr>
               </Thead>
               <Tbody>
-                <Tr>
-                  <Td>
+                <Tr color={'myGray.800'}>
+                  <Td pt={0}>
                     <Flex alignItems={'center'}>
                       <MyIcon
                         name={DatasetSearchModeMap[datasetTestItem.searchMode]?.icon as any}
@@ -458,45 +484,31 @@ const TestResults = React.memo(function TestResults({
                       {t(DatasetSearchModeMap[datasetTestItem.searchMode]?.title)}
                     </Flex>
                   </Td>
-                  <Td>{datasetTestItem.usingReRank ? '✅' : '❌'}</Td>
-                  <Td>{datasetTestItem.limit}</Td>
-                  <Td>{datasetTestItem.similarity}</Td>
+                  <Td pt={0}>{datasetTestItem.usingReRank ? '✅' : '❌'}</Td>
+                  <Td pt={0}>{datasetTestItem.limit}</Td>
+                  <Td pt={0}>{datasetTestItem.similarity}</Td>
                 </Tr>
               </Tbody>
             </Table>
           </TableContainer>
-          <Flex alignItems={'center'}>
-            <Box fontSize={'xl'} color={'myGray.600'}>
+
+          <Flex mt={5} mb={3} alignItems={'center'}>
+            <Flex fontSize={'xl'} color={'myGray.900'} alignItems={'center'}>
+              <MyIcon name={'common/resultLight'} w={'18px'} mr={2} />
               {t('core.dataset.test.Test Result')}
-            </Box>
+            </Flex>
             <MyTooltip label={t('core.dataset.test.test result tip')} forceShow>
               <QuestionOutlineIcon mx={2} color={'myGray.600'} cursor={'pointer'} fontSize={'lg'} />
             </MyTooltip>
             <Box>({datasetTestItem.duration})</Box>
           </Flex>
-          <Grid
-            mt={1}
-            gridTemplateColumns={[
-              'repeat(1,minmax(0, 1fr))',
-              'repeat(1,minmax(0, 1fr))',
-              'repeat(1,minmax(0, 1fr))',
-              'repeat(1,minmax(0, 1fr))',
-              'repeat(2,minmax(0, 1fr))'
-            ]}
-            gridGap={4}
-          >
+          <Box mt={1} gap={4}>
             {datasetTestItem?.results.map((item, index) => (
-              <Box
-                key={item.id}
-                p={2}
-                borderRadius={'sm'}
-                border={theme.borders.base}
-                _notLast={{ mb: 2 }}
-              >
+              <Box key={item.id} p={3} borderRadius={'lg'} bg={'myGray.100'} _notLast={{ mb: 2 }}>
                 <QuoteItem quoteItem={item} canViewSource />
               </Box>
             ))}
-          </Grid>
+          </Box>
         </>
       )}
     </>

projects/app/src/pages/dataset/list/index.tsx

@@ -20,9 +20,9 @@ import {
   delDatasetById,
   getDatasetPaths,
   putDatasetById,
-  postCreateDataset,
-  getCheckExportLimit
+  postCreateDataset
 } from '@/web/core/dataset/api';
+import { checkTeamExportDatasetLimit } from '@/web/support/user/api';
 import { useTranslation } from 'next-i18next';
 import Avatar from '@/components/Avatar';
 import MyIcon from '@fastgpt/web/components/common/Icon';
@@ -99,7 +99,7 @@ const Kb = () => {
   const { mutate: exportDataset } = useRequest({
     mutationFn: async (dataset: DatasetItemType) => {
       setLoading(true);
-      await getCheckExportLimit(dataset._id);
+      await checkTeamExportDatasetLimit(dataset._id);
       const a = document.createElement('a');
       a.href = `/api/core/dataset/exportAll?datasetId=${dataset._id}`;
       a.download = `${dataset.name}.csv`;

projects/app/src/service/moduleDispatch/agent/classifyQuestion.ts

@@ -68,7 +68,7 @@ export const dispatchClassifyQuestion = async (props: Props): Promise<CQResponse
   });
 
   return {
-    [result.key]: result.value,
+    [result.key]: true,
     [ModuleOutputKeyEnum.responseData]: {
       price: user.openaiAccount?.key ? 0 : total,
       model: modelName,

projects/app/src/service/moduleDispatch/chat/oneapi.ts

@@ -272,8 +272,8 @@ function filterQuote({
   });
 
   const quoteText =
-    filterQuoteQA.length > 0
-      ? `${filterQuoteQA.map((item, index) => getValue(item, index)).join('\n')}`
+    sortQuoteQAList.length > 0
+      ? `${sortQuoteQAList.map((item, index) => getValue(item, index)).join('\n')}`
       : '';
 
   return {

projects/app/src/web/core/dataset/api.ts

@@ -25,7 +25,10 @@ import type {
 } from '@/global/core/dataset/api.d';
 import type { PushDataResponse } from '@/global/core/api/datasetRes.d';
 import type { DatasetCollectionItemType } from '@fastgpt/global/core/dataset/type';
-import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant';
+import {
+  DatasetCollectionSyncResultEnum,
+  DatasetTypeEnum
+} from '@fastgpt/global/core/dataset/constant';
 import type { DatasetDataItemType } from '@fastgpt/global/core/dataset/type';
 import type { DatasetCollectionsListItemType } from '@/global/core/dataset/type.d';
 import { PagingData } from '@/types';
@@ -56,9 +59,6 @@ export const postWebsiteSync = (data: PostWebsiteSyncParams) =>
     timeout: 600000
   }).catch();
 
-export const getCheckExportLimit = (datasetId: string) =>
-  GET(`/core/dataset/checkExportLimit`, { datasetId });
-
 /* =========== search test ============ */
 export const postSearchText = (data: SearchTestProps) =>
   POST<SearchTestResponse>(`/core/dataset/searchTest`, data);
@@ -77,7 +77,9 @@ export const putDatasetCollectionById = (data: UpdateDatasetCollectionParams) =>
 export const delDatasetCollectionById = (params: { collectionId: string }) =>
   DELETE(`/core/dataset/collection/delete`, params);
 export const postLinkCollectionSync = (collectionId: string) =>
-  POST(`/core/dataset/collection/sync/link`, { collectionId });
+  POST<`${DatasetCollectionSyncResultEnum}`>(`/core/dataset/collection/sync/link`, {
+    collectionId
+  });
 
 /* =============================== data ==================================== */
 /* get dataset list */

projects/app/src/web/core/dataset/store/dataset.ts

@@ -13,6 +13,7 @@ import { defaultDatasetDetail } from '@/constants/dataset';
 import type { DatasetUpdateBody } from '@fastgpt/global/core/dataset/api.d';
 import { DatasetStatusEnum } from '@fastgpt/global/core/dataset/constant';
 import { postCreateTrainingBill } from '@/web/support/wallet/bill/api';
+import { checkTeamWebSyncLimit } from '@/web/support/user/api';
 
 type State = {
   allDatasets: DatasetListItemType[];
@@ -86,6 +87,8 @@ export const useDatasetStore = create<State>()(
           });
         },
         async startWebsiteSync() {
+          await checkTeamWebSyncLimit();
+
           const [_, billId] = await Promise.all([
             get().updateDataset({
               id: get().datasetDetail._id,

projects/app/src/web/support/user/api.ts

@@ -71,3 +71,8 @@ export const postLogin = ({ password, ...props }: PostLoginProps) =>
 export const loginOut = () => GET('/support/user/account/loginout');
 
 export const putUserInfo = (data: UserUpdateParams) => PUT('/support/user/account/update', data);
+
+/* team limit */
+export const checkTeamExportDatasetLimit = (datasetId: string) =>
+  GET(`/support/user/team/limit/exportDatasetLimit`, { datasetId });
+export const checkTeamWebSyncLimit = () => GET(`/support/user/team/limit/webSyncLimit`);

scripts/icon/index.js

@@ -33,7 +33,7 @@ app.get('/', (req, res) => {
 
   svgPaths.forEach((filePath) => {
     const name = filePath.split('.')[0];
-    iconHtml += `<div class="item" id="${name}" onclick="onclickCopy(${name})">
+    iconHtml += `<div class="item" id="${name}" onclick="onclickCopy('${name}')">
       <img src="/icons/${filePath}" width="30" height="30" /> 
       <div>${name}</div>
     </div>`;
@@ -76,16 +76,16 @@ app.get('/', (req, res) => {
   </body>
   <script>
     const onclickCopy = (name) => {
-      console.log(name.id)
+      console.log(name)
       try {
         if (navigator.clipboard) {
-          navigator.clipboard.writeText(name.id);
+          navigator.clipboard.writeText(name);
         } else {
           throw new Error('');
         }
       } catch (error) {
         const textarea = document.createElement('textarea');
-        textarea.value = name.id;
+        textarea.value = name;
         document.body.appendChild(textarea);
         textarea.select();
         document.execCommand('copy');