scripts: add delete-unused-keys (#2115)

* scripts: add delete-unused-keys

* update script

* fix keys

* fix ci

* update keys

* add i18n

* update

package.json

@@ -9,7 +9,8 @@
     "gen:theme-typings": "chakra-cli tokens packages/web/styles/theme.ts --out node_modules/.pnpm/node_modules/@chakra-ui/styled-system/dist/theming.types.d.ts",
     "postinstall": "sh ./scripts/postinstall.sh",
     "initIcon": "node ./scripts/icon/init.js",
-    "previewIcon": "node ./scripts/icon/index.js"
+    "previewIcon": "node ./scripts/icon/index.js",
+    "checkI18n": "node ./scripts/i18n/delete-unused-keys.js"
   },
   "devDependencies": {
     "@chakra-ui/cli": "^2.4.1",

packages/global/core/dataset/constants.ts

@@ -8,22 +8,22 @@ export enum DatasetTypeEnum {
 export const DatasetTypeMap = {
   [DatasetTypeEnum.folder]: {
     icon: 'common/folderFill',
-    label: 'Folder Dataset',
+    label: 'folder_dataset',
     collectionLabel: 'common.Folder'
   },
   [DatasetTypeEnum.dataset]: {
     icon: 'core/dataset/commonDataset',
-    label: 'Common Dataset',
+    label: 'common_dataset',
     collectionLabel: 'common.File'
   },
   [DatasetTypeEnum.websiteDataset]: {
     icon: 'core/dataset/websiteDataset',
-    label: 'Website Dataset',
+    label: 'website_dataset',
     collectionLabel: 'common.Website'
   },
   [DatasetTypeEnum.externalFile]: {
     icon: 'core/dataset/externalDataset',
-    label: 'External File',
+    label: 'external_file',
     collectionLabel: 'common.File'
   }
 };

packages/web/i18n/en/app.json

 {
-  "plugin_cost_per_times": "{{cost}}/per time",
+  "Run": "Run",
   "ai_settings": "AI Settings",
+  "app": {
+    "modules": {
+      "click to update": "click to update",
+      "has new version": "has new version"
+    }
+  },
   "app_detail": "App Details",
   "chat_debug": "Chat Debug",
-  "chat_logs_tips": "Logs will record online, shared and API (chatId required) conversation records for this app",
   "chat_logs": "Chat Logs",
-  "confirm_del_app_tip": "Confirm to delete this app and all its chat records?",
+  "chat_logs_tips": "Logs will record online, shared and API (chatId required) conversation records for this app",
   "confirm_copy_app_tip": "The system will create an application with the same configuration for you, but the permission will not be copied, please confirm!",
+  "confirm_del_app_tip": "Confirm to delete this app and all its chat records?",
   "confirm_delete_folder_tip": "Are you sure to delete this folder? All the following applications and corresponding chat records will be deleted, please confirm!",
   "copy_one_app": "Copy",
   "create_copy_success": "Create copy success",
@@ -24,39 +30,31 @@
   "logs_message_total": "Total Messages",
   "logs_title": "Title",
   "mark_count": "Marked Answer Count",
+  "module": {
+    "Confirm Sync": "The template will be updated to the latest template configuration. Fields that do not exist in the template will be deleted (including all custom fields). You are advised to make a copy of the node and then update the original node version.",
+    "Custom Title Tip": "This title will be displayed during the conversation",
+    "No Modules": "No plugins yet~",
+    "type": "\"{{type}}\" type\n{{description}}"
+  },
+  "modules": {
+    "Title is required": "Module name cannot be empty"
+  },
   "move_app": "Move app",
   "paste_config": "Paste Config",
+  "plugin_cost_per_times": "{{cost}}/per time",
   "plugin_dispatch": "Plugins",
   "plugin_dispatch_tip": "It is up to the model to decide which plug-ins to add additional capabilities to. If the plug-in is selected, the knowledge base call is also treated as a special plug-in.",
   "publish_channel": "Publish channel",
   "publish_success": "Publish success",
-  "Run": "Run",
   "search_app": "Search app",
   "setting_app": "Settings",
   "setting_plugin": "Setting plugin",
-  "transition_to_workflow": "Transition to workflow",
-  "transition_to_workflow_create_new_placeholder": "Create a new application instead of modifying the current one",
-  "transition_to_workflow_create_new_tip": "After converting to workflow, it will not be able to convert back to simple mode, please confirm!",
-  "app": {
-    "modules": {
-      "click to update": "click to update",
-      "has new version": "has new version"
-    }
-  },
-  "common": {
-    "plugin_cost_per_times": "{{cost}} points/times"
-  },
-  "module": {
-    "Confirm Sync": "The template will be updated to the latest template configuration. Fields that do not exist in the template will be deleted (including all custom fields). You are advised to make a copy of the node and then update the original node version.",
-    "Custom Title Tip": "This title will be displayed during the conversation",
-    "No Modules": "No plugins yet~"
-  },
-  "modules": {
-    "Title is required": "Module name cannot be empty"
-  },
   "template": {
     "simple_robot": "Simple Robot"
   },
+  "transition_to_workflow": "Transition to workflow",
+  "transition_to_workflow_create_new_placeholder": "Create a new application instead of modifying the current one",
+  "transition_to_workflow_create_new_tip": "After converting to workflow, it will not be able to convert back to simple mode, please confirm!",
   "type": {
     "All": "All",
     "Create http plugin tip": "Create plug-ins in batches using OpenAPI schema, compatible with GPTs format.",
@@ -74,4 +72,4 @@
   "version": {
     "Revert success": "Revert success"
   }
-}
\ No newline at end of file
+}

packages/web/i18n/en/chat.json

 {
   "chat_input_guide_lexicon_is_empty": "The lexicon has not been configured",
+  "config_input_guide": "Configure input boot",
   "config_input_guide_lexicon": "Config",
   "config_input_guide_lexicon_title": "Config lexicon",
   "csv_input_lexicon_tip": "Only CSV can be imported in batches. Click to download the template",

packages/web/i18n/en/dataset.json

@@ -12,6 +12,7 @@
   "external_read_url": "External read url",
   "external_read_url_tip": "You can configure the reading address of your file library. This allows users to read and authenticate. You can currently use the {{fileId}} variable to refer to the external file ID.",
   "external_url": "File read url",
+  "folder_dataset": "Folder",
   "rebuild_embedding_start_tip": "The task of switching index models has begun",
   "rebuilding_index_count": "Rebuilding count: {{count}}",
   "the_knowledge_base_has_indexes_that_are_being_trained_or_being_rebuilt": "The knowledge base has indexes that are being trained or being rebuilt",

packages/web/i18n/en/user.json

 {
   "permission": {
-    "Write": "Write"
+    "Manage": "administrator",
+    "Manage tip": "Team administrator, with full permissions",
+    "Read": "Read only",
+    "Read desc": "Members can only read related resources and cannot create new resources.",
+    "Write": "Write",
+    "Write tip": "In addition to readable resources, you can also create new resources"
   },
   "team": {
     "Add manager": "Add manager"

packages/web/i18n/zh/app.json

 {
-  "plugin_cost_per_times": "{{cost}}/次",
+  "Run": "运行",
   "ai_settings": "AI 配置",
+  "app": {
+    "modules": {
+      "click to update": "点击更新",
+      "has new version": "有新版本"
+    }
+  },
   "app_detail": "应用详情",
   "chat_debug": "调试预览",
-  "chat_logs_tips": "日志会记录该应用的在线、分享和 API（需填写 chatId）对话记录",
   "chat_logs": "对话日志",
-  "confirm_del_app_tip": "确认删除该应用及其所有聊天记录？",
+  "chat_logs_tips": "日志会记录该应用的在线、分享和 API（需填写 chatId）对话记录",
   "confirm_copy_app_tip": "系统将为您创建一个相同配置应用，但权限不会进行复制，请确认！",
+  "confirm_del_app_tip": "确认删除该应用及其所有聊天记录？",
   "confirm_delete_folder_tip": "确认删除该文件夹？将会删除它下面所有应用及对应的聊天记录，请确认！",
   "copy_one_app": "创建副本",
   "create_copy_success": "创建副本成功",
@@ -24,39 +30,31 @@
   "logs_message_total": "消息总数",
   "logs_title": "标题",
   "mark_count": "标注答案数量",
+  "module": {
+    "Confirm Sync": "将会更新至最新的模板配置，不存在模板中的字段将会被删除（包括所有自定义字段），建议您先复制一份节点，再更新原来节点的版本。",
+    "Custom Title Tip": "该标题名字会展示在对话过程中",
+    "No Modules": "没找到插件",
+    "type": "\"{{type}}\"类型\n{{description}}"
+  },
+  "modules": {
+    "Title is required": "模块名不能为空"
+  },
   "move_app": "移动应用",
   "paste_config": "粘贴配置",
+  "plugin_cost_per_times": "{{cost}}/次",
   "plugin_dispatch": "插件调用",
   "plugin_dispatch_tip": "给模型附加额外的能力，具体调用哪些插件，将由模型自主决定。\n若选择了插件，知识库调用将自动作为一个特殊的插件。",
   "publish_channel": "发布渠道",
   "publish_success": "发布成功",
-  "Run": "运行",
   "search_app": "搜索应用",
   "setting_app": "应用配置",
   "setting_plugin": "插件配置",
-  "transition_to_workflow": "转成工作流",
-  "transition_to_workflow_create_new_placeholder": "创建一个新的应用，而不是修改当前应用",
-  "transition_to_workflow_create_new_tip": "转化成工作流后，将无法转化回简易模式，请确认！",
-  "app": {
-    "modules": {
-      "click to update": "点击更新",
-      "has new version": "有新版本"
-    }
-  },
-  "common": {
-    "plugin_cost_per_times": "{{cost}}积分/次"
-  },
-  "module": {
-    "Confirm Sync": "将会更新至最新的模板配置，不存在模板中的字段将会被删除（包括所有自定义字段），建议您先复制一份节点，再更新原来节点的版本。",
-    "Custom Title Tip": "该标题名字会展示在对话过程中",
-    "No Modules": "没找到插件"
-  },
-  "modules": {
-    "Title is required": "模块名不能为空"
-  },
   "template": {
     "simple_robot": "简易机器人"
   },
+  "transition_to_workflow": "转成工作流",
+  "transition_to_workflow_create_new_placeholder": "创建一个新的应用，而不是修改当前应用",
+  "transition_to_workflow_create_new_tip": "转化成工作流后，将无法转化回简易模式，请确认！",
   "type": {
     "All": "全部",
     "Create http plugin tip": "通过 OpenAPI Schema 批量创建插件，兼容 GPTs 格式。",
@@ -74,4 +72,4 @@
   "version": {
     "Revert success": "回滚成功"
   }
-}
\ No newline at end of file
+}

packages/web/i18n/zh/dataset.json

@@ -12,6 +12,7 @@
   "external_read_url": "外部预览地址",
   "external_read_url_tip": "可以配置你文件库的阅读地址。便于对用户进行阅读鉴权操作。目前可以使用 {{fileId}} 变量来指代外部文件 ID。",
   "external_url": "文件访问 URL",
+  "folder_dataset": "文件夹",
   "rebuild_embedding_start_tip": "切换索引模型任务已开始",
   "rebuilding_index_count": "重建中索引数量：{{count}}",
   "the_knowledge_base_has_indexes_that_are_being_trained_or_being_rebuilt": "知识库有训练中或正在重建的索引",

projects/app/src/pages/app/detail/components/WorkflowComponents/Flow/nodes/render/RenderToolInput/index.tsx

@@ -45,7 +45,7 @@ const RenderToolInput = ({
   return (
     <>
       <HStack mb={2} justifyContent={'space-between'}>
-        <IOTitle text={t('common:core.module.tool.Tool input')} mb={0} />
+        <IOTitle text={t('workflow:tool_input')} mb={0} />
         {dynamicInput && (
           <Button
             variant={'whiteBase'}

scripts/i18n/delete-unused-keys.js

+const fs = require('fs').promises
+const path = require('path')
+
+// 配置项
+const CONFIG = {
+  i18nDirectory: path.join(__dirname, '../../packages/web/i18n'),
+  sourceDirectories: ['../../packages', '../../projects/app'].map(dir => path.join(__dirname, dir)),
+  fileExtensions: ['.ts', '.tsx', '.js', '.jsx'],
+  ignoreDirectories: ['node_modules', '.next', 'public', 'i18n']
+}
+
+// 从文件中加载 JSON
+const loadJSON = async (filePath) => {
+  try {
+    const data = await fs.readFile(filePath, 'utf8')
+    return JSON.parse(data)
+  } catch (error) {
+    console.error(`读取 JSON 文件 ${filePath} 时出错:`, error)
+    return null
+  }
+}
+
+// 递归提取 JSON 对象的所有键
+const extractKeysFromJSON = (obj, parentKey = '') => {
+  let keys = []
+  for (const [key, value] of Object.entries(obj)) {
+    const fullKey = parentKey ? `${parentKey}.${key}` : key
+    if (typeof value === 'object' && value !== null) {
+      keys = keys.concat(extractKeysFromJSON(value, fullKey))
+    } else {
+      keys.push(fullKey)
+    }
+  }
+  return keys
+}
+
+// 递归获取文件夹中的所有 JSON 文件
+const getAllJSONFiles = async (dir) => {
+  let results = []
+  const list = await fs.readdir(dir)
+  await Promise.all(list.map(async (file) => {
+    const filePath = path.join(dir, file)
+    const stat = await fs.stat(filePath)
+    if (stat.isDirectory()) {
+      if (!CONFIG.ignoreDirectories.includes(file)) {
+        results = results.concat(await getAllJSONFiles(filePath))
+      }
+    } else if (filePath.endsWith('.json')) {
+      results.push(filePath)
+    }
+  }))
+  return results
+}
+
+// 提取文件夹中所有 JSON 文件的键
+const extractKeysFromDirectory = async (dir) => {
+  let allKeys = new Set()
+
+  const subDirs = await fs.readdir(dir)
+  await Promise.all(subDirs.map(async (subDir) => {
+    const subDirPath = path.join(dir, subDir)
+    const stat = await fs.stat(subDirPath)
+    if (stat.isDirectory() && !CONFIG.ignoreDirectories.includes(subDir)) {
+      const files = await getAllJSONFiles(subDirPath)
+      await Promise.all(files.map(async (file) => {
+        const jsonObject = await loadJSON(file)
+        if (jsonObject) {
+          const keys = extractKeysFromJSON(jsonObject)
+          keys.forEach(key => allKeys.add(key))
+        }
+      }))
+    }
+  }))
+
+  return Array.from(allKeys)
+}
+
+// 检查键是否在内容中使用
+const isKeyUsedInContent = (content, key) => {
+  const regex = new RegExp(`\\b${key}\\b`, 'g')
+  return regex.test(content)
+}
+
+// 递归在文件夹中搜索键
+const searchKeysInFiles = async (dir, keys, usedKeys) => {
+  const list = await fs.readdir(dir)
+  await Promise.all(list.map(async (file) => {
+    const filePath = path.join(dir, file)
+    const stat = await fs.stat(filePath)
+    if (stat.isDirectory()) {
+      if (!CONFIG.ignoreDirectories.includes(file)) {
+        await searchKeysInFiles(filePath, keys, usedKeys)
+      }
+    } else if (CONFIG.fileExtensions.includes(path.extname(file))) {
+      const data = await fs.readFile(filePath, 'utf8')
+      keys.forEach(key => {
+        if (isKeyUsedInContent(data, key)) {
+          usedKeys.add(key)
+        }
+      })
+    }
+  }))
+}
+
+// 从扁平键路径恢复嵌套对象
+const restoreNestedStructure = (keys) => {
+  const result = {}
+  keys.forEach(key => {
+    const parts = key.split('.')
+    let current = result
+    parts.forEach((part, index) => {
+      if (!current[part]) {
+        current[part] = index === parts.length - 1 ? null : {}
+      }
+      current = current[part]
+    })
+  })
+  return result
+}
+
+// 递归删除未使用的键
+const removeUnusedKeys = (target, unusedKeys) => {
+  for (const [key, value] of Object.entries(unusedKeys)) {
+    if (value === null) {
+      delete target[key]
+    } else if (typeof target[key] === 'object' && target[key] !== null) {
+      removeUnusedKeys(target[key], value)
+      if (Object.keys(target[key]).length === 0) {
+        delete target[key]
+      }
+    }
+  }
+}
+
+// 处理 JSON 文件
+const processJSONFile = async (filePath, unusedKeys) => {
+  try {
+    const jsonObject = await loadJSON(filePath)
+    if (jsonObject) {
+      removeUnusedKeys(jsonObject, unusedKeys)
+      await fs.writeFile(filePath, JSON.stringify(jsonObject, null, 2), 'utf8')
+      console.log(`已处理文件 ${filePath}`)
+    }
+  } catch (error) {
+    console.error(`处理文件 ${filePath} 时出错:`, error)
+  }
+}
+
+// 递归处理目录中的所有 JSON 文件
+const processJSONFilesInDirectory = async (dir, unusedKeys) => {
+  const list = await fs.readdir(dir)
+  await Promise.all(list.map(async (file) => {
+    const filePath = path.join(dir, file)
+    const stat = await fs.stat(filePath)
+    if (stat.isDirectory() && !CONFIG.ignoreDirectories.includes(file)) {
+      await processJSONFilesInDirectory(filePath, unusedKeys)
+    } else if (stat.isFile() && file.endsWith('.json')) {
+      await processJSONFile(filePath, unusedKeys)
+    }
+  }))
+}
+
+// 将 keys 写入 JSON 文件
+const writeKeysToFile = async (filePath, keys) => {
+  try {
+    await fs.writeFile(filePath, JSON.stringify(keys, null, 2), 'utf8')
+    console.log(`已将 keys 写入文件 ${filePath}`)
+  } catch (error) {
+    console.error(`写入文件 ${filePath} 时出错:`, error)
+  }
+}
+
+// 主函数
+const main = async () => {
+  const allKeys = await extractKeysFromDirectory(CONFIG.i18nDirectory)
+  // await writeKeysToFile(path.join(__dirname, 'allKeys.json'), allKeys)
+  const usedKeys = new Set()
+  await Promise.all(CONFIG.sourceDirectories.map(dir => searchKeysInFiles(dir, allKeys, usedKeys)))
+
+  const unusedKeys = allKeys.filter(key => !usedKeys.has(key))
+  // await writeKeysToFile(path.join(__dirname, 'unusedKeys.json'), unusedKeys)
+  console.log(unusedKeys)
+
+  const nestedUnusedKeys = restoreNestedStructure(unusedKeys)
+  await processJSONFilesInDirectory(CONFIG.i18nDirectory, nestedUnusedKeys)
+}
+
+main().catch(err => console.error('执行过程中出错:', err))
\ No newline at end of file