feat: custom dataset split sign (#4221)

* feat: custom dataset split sign * feat: custom dataset split sign

feat: custom dataset split sign (#4221)
* feat: custom dataset split sign * feat: custom dataset split sign
ec30d792 · Archer · archer · cb29076e · ec30d792 · ec30d792
Commit ec30d792 authored Mar 18, 2025 by Archer Committed by archer Mar 27, 2025
9 changed files
--- a/docSite/content/zh-cn/docs/development/upgrading/492.md
+++ b/docSite/content/zh-cn/docs/development/upgrading/492.md
+---
+title: 'V4.9.2(进行中)'
+description: 'FastGPT V4.9.2 更新说明'
+icon: 'upgrade'
+draft: false
+toc: true
+weight: 799
+---
+## 🚀 新增内容
+1. 知识库分块增加自定义分隔符预设值，同时支持自定义换行符分割。
+## ⚙️ 优化
+1. 导出对话日志时，支持导出成员名。
+## 🐛 修复
--- a/packages/global/common/string/textSplitter.ts
+++ b/packages/global/common/string/textSplitter.ts
 import { getErrText } from '../error/utils';
-import { replaceRegChars } from './tools';
 export const CUSTOM_SPLIT_SIGN = '-----CUSTOM_SPLIT_SIGN-----';
@@ -115,9 +114,10 @@ const commonSplit = (props: SplitProps): SplitResponse => {
  // The larger maxLen is, the next sentence is less likely to trigger splitting
  const markdownIndex = 4;
  const forbidOverlapIndex = 8;
-  const stepReges: { reg: RegExp; maxLen: number }[] = [
+  const stepReges: { reg: RegExp | string; maxLen: number }[] = [
    ...customReg.map((text) => ({
-      reg: new RegExp(`(${replaceRegChars(text)})`, 'g'),
+      reg: text.replaceAll('\\n', '\n'),
      maxLen: chunkLen * 1.4
    })),
    { reg: /^(#\s[^\n]+\n)/gm, maxLen: chunkLen * 1.2 },
@@ -161,17 +161,32 @@ const commonSplit = (props: SplitProps): SplitResponse => {
    const { reg } = stepReges[step];
-    const splitTexts = text
+    const replaceText = (() => {
-      .replace(
+      if (typeof reg === 'string') {
+        let tmpText = text;
+        reg.split('|').forEach((itemReg) => {
+          tmpText = tmpText.replaceAll(
+            itemReg,
+            (() => {
+              if (isCustomStep) return splitMarker;
+              if (isMarkdownSplit) return `${splitMarker}$1`;
+              return `$1${splitMarker}`;
+            })()
+          );
+        });
+        return tmpText;
+      }
+      return text.replace(
        reg,
        (() => {
          if (isCustomStep) return splitMarker;
          if (isMarkdownSplit) return `${splitMarker}$1`;
          return `$1${splitMarker}`;
        })()
-      )
+      );
-      .split(`${splitMarker}`)
+    })();
-      .filter((part) => part.trim());
+    const splitTexts = replaceText.split(splitMarker).filter((part) => part.trim());
    return splitTexts
      .map((text) => {

--- a/packages/web/i18n/en/common.json
+++ b/packages/web/i18n/en/common.json
@@ -570,7 +570,6 @@
  "core.dataset.import.Custom process desc": "Customize segmentation and preprocessing rules",
  "core.dataset.import.Custom prompt": "Custom Prompt",
  "core.dataset.import.Custom split char": "Custom Separator",
-  "core.dataset.import.Custom split char Tips": "Allows you to segment based on custom separators. Usually used for pre-processed data, using specific separators for precise segmentation.",
  "core.dataset.import.Custom text": "Custom Text",
  "core.dataset.import.Custom text desc": "Manually enter a piece of text as a dataset",
  "core.dataset.import.Data process params": "Data Processing Parameters",

--- a/packages/web/i18n/en/dataset.json
+++ b/packages/web/i18n/en/dataset.json
@@ -25,6 +25,7 @@
  "core.dataset.import.Adjust parameters": "Adjust parameters",
  "custom_data_process_params": "Custom",
  "custom_data_process_params_desc": "Customize data processing rules",
+  "custom_split_sign_tip": "Allows you to chunk according to custom delimiters. \nUsually used for processed data, using specific separators for precise chunking. \nYou can use the | symbol to represent multiple splitters, such as: \".|.\" to represent a period in Chinese and English.\n\nTry to avoid using special symbols related to regular, such as: * () [] {}, etc.",
  "data.ideal_chunk_length": "ideal block length",
  "data_amount": "{{dataAmount}} Datas, {{indexAmount}} Indexes",
  "data_index_num": "Index {{index}}",
@@ -86,6 +87,14 @@
  "retain_collection": "Adjust Training Parameters",
  "retrain_task_submitted": "The retraining task has been submitted",
  "same_api_collection": "The same API set exists",
+  "split_sign_break": "1 newline character",
+  "split_sign_break2": "2 newline characters",
+  "split_sign_custom": "Customize",
+  "split_sign_exclamatiob": "exclamation mark",
+  "split_sign_null": "Not set",
+  "split_sign_period": "period",
+  "split_sign_question": "question mark",
+  "split_sign_semicolon": "semicolon",
  "start_sync_website_tip": "Confirm to start synchronizing data? \nThe old data will be deleted and retrieved again, please confirm!",
  "sync_collection_failed": "Synchronization collection error, please check whether the source file can be accessed normally",
  "sync_schedule": "Timing synchronization",

--- a/packages/web/i18n/zh-CN/common.json
+++ b/packages/web/i18n/zh-CN/common.json
@@ -574,7 +574,6 @@
  "core.dataset.import.Custom process desc": "自定义设置数据处理规则",
  "core.dataset.import.Custom prompt": "自定义提示词",
  "core.dataset.import.Custom split char": "自定义分隔符",
-  "core.dataset.import.Custom split char Tips": "允许你根据自定义的分隔符进行分块。通常用于已处理好的数据，使用特定的分隔符来精确分块。",
  "core.dataset.import.Custom text": "自定义文本",
  "core.dataset.import.Custom text desc": "手动输入一段文本作为数据集",
  "core.dataset.import.Data process params": "数据处理参数",

--- a/packages/web/i18n/zh-CN/dataset.json
+++ b/packages/web/i18n/zh-CN/dataset.json
@@ -25,6 +25,7 @@
  "core.dataset.import.Adjust parameters": "调整参数",
  "custom_data_process_params": "自定义",
  "custom_data_process_params_desc": "自定义设置数据处理规则",
+  "custom_split_sign_tip": "允许你根据自定义的分隔符进行分块。通常用于已处理好的数据，使用特定的分隔符来精确分块。可以使用 | 符号表示多个分割符，例如：“。|.” 表示中英文句号。\n尽量避免使用正则相关特殊符号，例如: * () [] {} 等。",
  "data.ideal_chunk_length": "理想分块长度",
  "data_amount": "{{dataAmount}} 组数据, {{indexAmount}} 组索引",
  "data_index_num": "索引 {{index}}",
@@ -86,6 +87,14 @@
  "retain_collection": "调整训练参数",
  "retrain_task_submitted": "重新训练任务已提交",
  "same_api_collection": "存在相同的 API 集合",
+  "split_sign_break": "1 个换行符",
+  "split_sign_break2": "2 个换行符",
+  "split_sign_custom": "自定义",
+  "split_sign_exclamatiob": "感叹号",
+  "split_sign_null": "不设置",
+  "split_sign_period": "句号",
+  "split_sign_question": "问号",
+  "split_sign_semicolon": "分号",
  "start_sync_website_tip": "确认开始同步数据？将会删除旧数据后重新获取，请确认！",
  "sync_collection_failed": "同步集合错误，请检查是否能正常访问源文件",
  "sync_schedule": "定时同步",

--- a/packages/web/i18n/zh-Hant/common.json
+++ b/packages/web/i18n/zh-Hant/common.json
@@ -569,7 +569,6 @@
  "core.dataset.import.Custom process desc": "自訂設定資料處理規則",
  "core.dataset.import.Custom prompt": "自訂提示詞",
  "core.dataset.import.Custom split char": "自訂分隔符",
-  "core.dataset.import.Custom split char Tips": "允許您根據自訂的分隔符進行分割。通常用於已處理好的資料，使用特定的分隔符來精確分割。",
  "core.dataset.import.Custom text": "自訂文字",
  "core.dataset.import.Custom text desc": "手動輸入一段文字作為資料集",
  "core.dataset.import.Data process params": "資料處理參數",

--- a/packages/web/i18n/zh-Hant/dataset.json
+++ b/packages/web/i18n/zh-Hant/dataset.json
@@ -25,6 +25,7 @@
  "core.dataset.import.Adjust parameters": "調整參數",
  "custom_data_process_params": "自訂",
  "custom_data_process_params_desc": "自訂資料處理規則",
+  "custom_split_sign_tip": "允許你根據自定義的分隔符進行分塊。\n通常用於已處理好的數據，使用特定的分隔符來精確分塊。\n可以使用 | 符號表示多個分割符，例如：“。|.” 表示中英文句號。\n\n盡量避免使用正則相關特殊符號，例如: * () [] {} 等。",
  "data.ideal_chunk_length": "理想分塊長度",
  "data_amount": "{{dataAmount}} 組數據, {{indexAmount}} 組索引",
  "data_index_num": "索引 {{index}}",
@@ -86,6 +87,14 @@
  "retain_collection": "調整訓練參數",
  "retrain_task_submitted": "重新訓練任務已提交",
  "same_api_collection": "存在相同的 API 集合",
+  "split_sign_break": "1 個換行符",
+  "split_sign_break2": "2 個換行符",
+  "split_sign_custom": "自定義",
+  "split_sign_exclamatiob": "驚嘆號",
+  "split_sign_null": "不設置",
+  "split_sign_period": "句號",
+  "split_sign_question": "問號",
+  "split_sign_semicolon": "分號",
  "start_sync_website_tip": "確認開始同步資料？\n將會刪除舊資料後重新獲取，請確認！",
  "sync_collection_failed": "同步集合錯誤，請檢查是否能正常存取來源文件",
  "sync_schedule": "定時同步",

--- a/projects/app/src/pageComponents/dataset/detail/Import/commonProgress/DataProcess.tsx
+++ b/projects/app/src/pageComponents/dataset/detail/Import/commonProgress/DataProcess.tsx
-import React, { useCallback, useEffect, useMemo, useRef } from 'react';
+import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
 import {
  Box,
  Flex,
@@ -36,6 +36,7 @@ import MyNumberInput from '@fastgpt/web/components/common/Input/NumberInput';
 import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
 import { shadowLight } from '@fastgpt/web/styles/theme';
 import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
+import MySelect from '@fastgpt/web/components/common/MySelect';
 function DataProcess() {
  const { t } = useTranslation();
@@ -44,18 +45,39 @@ function DataProcess() {
  const { goToNext, processParamsForm, chunkSizeField, minChunkSize, maxChunkSize } =
    useContextSelector(DatasetImportContext, (v) => v);
  const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
-  const { setValue, register, watch } = processParamsForm;
+  const { setValue, register, watch, getValues } = processParamsForm;
  const trainingType = watch('trainingType');
  const chunkSettingMode = watch('chunkSettingMode');
-  const qaPrompt = watch('qaPrompt');
+  const qaPrompt = watch('qaPrompt');
  const {
    isOpen: isOpenCustomPrompt,
    onOpen: onOpenCustomPrompt,
    onClose: onCloseCustomPrompt
  } = useDisclosure();
+  const customSplitList = [
+    { label: t('dataset:split_sign_null'), value: '' },
+    { label: t('dataset:split_sign_break'), value: '\\n' },
+    { label: t('dataset:split_sign_break2'), value: '\\n\\n' },
+    { label: t('dataset:split_sign_period'), value: '.|。' },
+    { label: t('dataset:split_sign_exclamatiob'), value: '!|！' },
+    { label: t('dataset:split_sign_question'), value: '?|？' },
+    { label: t('dataset:split_sign_semicolon'), value: ';|；' },
+    { label: '=====', value: '=====' },
+    { label: t('dataset:split_sign_custom'), value: 'Other' }
+  ];
+  const [customListSelectValue, setCustomListSelectValue] = useState(getValues('customSplitChar'));
+  useEffect(() => {
+    if (customListSelectValue === 'Other') {
+      setValue('customSplitChar', '');
+    } else {
+      setValue('customSplitChar', customListSelectValue);
+    }
+  }, [customListSelectValue, setValue]);
  const trainingModeList = useMemo(() => {
    const list = Object.entries(DatasetCollectionDataProcessModeMap);
    return list
@@ -248,19 +270,33 @@ function DataProcess() {
                          <Box mt={3}>
                            <Box>
                              {t('common:core.dataset.import.Custom split char')}
-                              <QuestionTip
+                              <QuestionTip label={t('dataset:custom_split_sign_tip')} />
-                                label={t('common:core.dataset.import.Custom split char Tips')}
+                            </Box>
+                            <HStack mt={1}>
+                              <Box flex={'1 0 0'}>
+                                <MySelect<string>
+                                  list={customSplitList}
+                                  size={'sm'}
+                                  bg={'myGray.50'}
+                                  value={customListSelectValue}
+                                  h={'32px'}
+                                  onChange={(val) => {
+                                    setCustomListSelectValue(val);
+                                  }}
                                />
                              </Box>
-                            <Box mt={1}>
+                              {customListSelectValue === 'Other' && (
                                <Input
+                                  flex={'1 0 0'}
+                                  h={'32px'}
                                  size={'sm'}
                                  bg={'myGray.50'}
-                                defaultValue={''}
                                  placeholder="\n;======;==SPLIT=="
                                  {...register('customSplitChar')}
                                />
-                            </Box>
+                              )}
+                            </HStack>
                          </Box>
                          {showQAPromptInput && (