scripts: add delete-unused-keys (#2115)

* scripts: add delete-unused-keys * update script * fix keys * fix ci * update keys * add i18n * update

scripts: add delete-unused-keys (#2115)
* scripts: add delete-unused-keys * update script * fix keys * fix ci * update keys * add i18n * update
991398b8 · jingyang · GitHub · f4525546 · 991398b8 · 991398b8
Commit 991398b8 authored Jul 22, 2024 by jingyang Committed by GitHub Jul 22, 2024
12 changed files
--- a/package.json
+++ b/package.json
@@ -9,7 +9,8 @@
    "gen:theme-typings": "chakra-cli tokens packages/web/styles/theme.ts --out node_modules/.pnpm/node_modules/@chakra-ui/styled-system/dist/theming.types.d.ts",
    "postinstall": "sh ./scripts/postinstall.sh",
    "initIcon": "node ./scripts/icon/init.js",
-    "previewIcon": "node ./scripts/icon/index.js"
+    "previewIcon": "node ./scripts/icon/index.js",
+    "checkI18n": "node ./scripts/i18n/delete-unused-keys.js"
  },
  "devDependencies": {
    "@chakra-ui/cli": "^2.4.1",

--- a/packages/global/core/dataset/constants.ts
+++ b/packages/global/core/dataset/constants.ts
@@ -8,22 +8,22 @@ export enum DatasetTypeEnum {
 export const DatasetTypeMap = {
  [DatasetTypeEnum.folder]: {
    icon: 'common/folderFill',
-    label: 'Folder Dataset',
+    label: 'folder_dataset',
    collectionLabel: 'common.Folder'
  },
  [DatasetTypeEnum.dataset]: {
    icon: 'core/dataset/commonDataset',
-    label: 'Common Dataset',
+    label: 'common_dataset',
    collectionLabel: 'common.File'
  },
  [DatasetTypeEnum.websiteDataset]: {
    icon: 'core/dataset/websiteDataset',
-    label: 'Website Dataset',
+    label: 'website_dataset',
    collectionLabel: 'common.Website'
  },
  [DatasetTypeEnum.externalFile]: {
    icon: 'core/dataset/externalDataset',
-    label: 'External File',
+    label: 'external_file',
    collectionLabel: 'common.File'
  }
 };

--- a/packages/web/i18n/en/app.json
+++ b/packages/web/i18n/en/app.json
 {
-  "plugin_cost_per_times": "{{cost}}/per time",
+  "Run": "Run",
  "ai_settings": "AI Settings",
+  "app": {
+    "modules": {
+      "click to update": "click to update",
+      "has new version": "has new version"
+    }
+  },
  "app_detail": "App Details",
  "chat_debug": "Chat Debug",
-  "chat_logs_tips": "Logs will record online, shared and API (chatId required) conversation records for this app",
  "chat_logs": "Chat Logs",
-  "confirm_del_app_tip": "Confirm to delete this app and all its chat records?",
+  "chat_logs_tips": "Logs will record online, shared and API (chatId required) conversation records for this app",
  "confirm_copy_app_tip": "The system will create an application with the same configuration for you, but the permission will not be copied, please confirm!",
+  "confirm_del_app_tip": "Confirm to delete this app and all its chat records?",
  "confirm_delete_folder_tip": "Are you sure to delete this folder? All the following applications and corresponding chat records will be deleted, please confirm!",
  "copy_one_app": "Copy",
  "create_copy_success": "Create copy success",
@@ -24,39 +30,31 @@
  "logs_message_total": "Total Messages",
  "logs_title": "Title",
  "mark_count": "Marked Answer Count",
+  "module": {
+    "Confirm Sync": "The template will be updated to the latest template configuration. Fields that do not exist in the template will be deleted (including all custom fields). You are advised to make a copy of the node and then update the original node version.",
+    "Custom Title Tip": "This title will be displayed during the conversation",
+    "No Modules": "No plugins yet~",
+    "type": "\"{{type}}\" type\n{{description}}"
+  },
+  "modules": {
+    "Title is required": "Module name cannot be empty"
+  },
  "move_app": "Move app",
  "paste_config": "Paste Config",
+  "plugin_cost_per_times": "{{cost}}/per time",
  "plugin_dispatch": "Plugins",
  "plugin_dispatch_tip": "It is up to the model to decide which plug-ins to add additional capabilities to. If the plug-in is selected, the knowledge base call is also treated as a special plug-in.",
  "publish_channel": "Publish channel",
  "publish_success": "Publish success",
-  "Run": "Run",
  "search_app": "Search app",
  "setting_app": "Settings",
  "setting_plugin": "Setting plugin",
-  "transition_to_workflow": "Transition to workflow",
-  "transition_to_workflow_create_new_placeholder": "Create a new application instead of modifying the current one",
-  "transition_to_workflow_create_new_tip": "After converting to workflow, it will not be able to convert back to simple mode, please confirm!",
-  "app": {
-    "modules": {
-      "click to update": "click to update",
-      "has new version": "has new version"
-    }
-  },
-  "common": {
-    "plugin_cost_per_times": "{{cost}} points/times"
-  },
-  "module": {
-    "Confirm Sync": "The template will be updated to the latest template configuration. Fields that do not exist in the template will be deleted (including all custom fields). You are advised to make a copy of the node and then update the original node version.",
-    "Custom Title Tip": "This title will be displayed during the conversation",
-    "No Modules": "No plugins yet~"
-  },
-  "modules": {
-    "Title is required": "Module name cannot be empty"
-  },
  "template": {
    "simple_robot": "Simple Robot"
  },
+  "transition_to_workflow": "Transition to workflow",
+  "transition_to_workflow_create_new_placeholder": "Create a new application instead of modifying the current one",
+  "transition_to_workflow_create_new_tip": "After converting to workflow, it will not be able to convert back to simple mode, please confirm!",
  "type": {
    "All": "All",
    "Create http plugin tip": "Create plug-ins in batches using OpenAPI schema, compatible with GPTs format.",

--- a/packages/web/i18n/en/chat.json
+++ b/packages/web/i18n/en/chat.json
 {
  "chat_input_guide_lexicon_is_empty": "The lexicon has not been configured",
+  "config_input_guide": "Configure input boot",
  "config_input_guide_lexicon": "Config",
  "config_input_guide_lexicon_title": "Config lexicon",
  "csv_input_lexicon_tip": "Only CSV can be imported in batches. Click to download the template",

--- a/packages/web/i18n/en/common.json
+++ b/packages/web/i18n/en/common.json
--- a/packages/web/i18n/en/dataset.json
+++ b/packages/web/i18n/en/dataset.json
@@ -12,6 +12,7 @@
  "external_read_url": "External read url",
  "external_read_url_tip": "You can configure the reading address of your file library. This allows users to read and authenticate. You can currently use the {{fileId}} variable to refer to the external file ID.",
  "external_url": "File read url",
+  "folder_dataset": "Folder",
  "rebuild_embedding_start_tip": "The task of switching index models has begun",
  "rebuilding_index_count": "Rebuilding count: {{count}}",
  "the_knowledge_base_has_indexes_that_are_being_trained_or_being_rebuilt": "The knowledge base has indexes that are being trained or being rebuilt",

--- a/packages/web/i18n/en/user.json
+++ b/packages/web/i18n/en/user.json
 {
  "permission": {
-    "Write": "Write"
+    "Manage": "administrator",
+    "Manage tip": "Team administrator, with full permissions",
+    "Read": "Read only",
+    "Read desc": "Members can only read related resources and cannot create new resources.",
+    "Write": "Write",
+    "Write tip": "In addition to readable resources, you can also create new resources"
  },
  "team": {
    "Add manager": "Add manager"

--- a/packages/web/i18n/zh/app.json
+++ b/packages/web/i18n/zh/app.json
 {
-  "plugin_cost_per_times": "{{cost}}/次",
+  "Run": "运行",
  "ai_settings": "AI 配置",
+  "app": {
+    "modules": {
+      "click to update": "点击更新",
+      "has new version": "有新版本"
+    }
+  },
  "app_detail": "应用详情",
  "chat_debug": "调试预览",
-  "chat_logs_tips": "日志会记录该应用的在线、分享和 API（需填写 chatId）对话记录",
  "chat_logs": "对话日志",
-  "confirm_del_app_tip": "确认删除该应用及其所有聊天记录？",
+  "chat_logs_tips": "日志会记录该应用的在线、分享和 API（需填写 chatId）对话记录",
  "confirm_copy_app_tip": "系统将为您创建一个相同配置应用，但权限不会进行复制，请确认！",
+  "confirm_del_app_tip": "确认删除该应用及其所有聊天记录？",
  "confirm_delete_folder_tip": "确认删除该文件夹？将会删除它下面所有应用及对应的聊天记录，请确认！",
  "copy_one_app": "创建副本",
  "create_copy_success": "创建副本成功",
@@ -24,39 +30,31 @@
  "logs_message_total": "消息总数",
  "logs_title": "标题",
  "mark_count": "标注答案数量",
+  "module": {
+    "Confirm Sync": "将会更新至最新的模板配置，不存在模板中的字段将会被删除（包括所有自定义字段），建议您先复制一份节点，再更新原来节点的版本。",
+    "Custom Title Tip": "该标题名字会展示在对话过程中",
+    "No Modules": "没找到插件",
+    "type": "\"{{type}}\"类型\n{{description}}"
+  },
+  "modules": {
+    "Title is required": "模块名不能为空"
+  },
  "move_app": "移动应用",
  "paste_config": "粘贴配置",
+  "plugin_cost_per_times": "{{cost}}/次",
  "plugin_dispatch": "插件调用",
  "plugin_dispatch_tip": "给模型附加额外的能力，具体调用哪些插件，将由模型自主决定。\n若选择了插件，知识库调用将自动作为一个特殊的插件。",
  "publish_channel": "发布渠道",
  "publish_success": "发布成功",
-  "Run": "运行",
  "search_app": "搜索应用",
  "setting_app": "应用配置",
  "setting_plugin": "插件配置",
-  "transition_to_workflow": "转成工作流",
-  "transition_to_workflow_create_new_placeholder": "创建一个新的应用，而不是修改当前应用",
-  "transition_to_workflow_create_new_tip": "转化成工作流后，将无法转化回简易模式，请确认！",
-  "app": {
-    "modules": {
-      "click to update": "点击更新",
-      "has new version": "有新版本"
-    }
-  },
-  "common": {
-    "plugin_cost_per_times": "{{cost}}积分/次"
-  },
-  "module": {
-    "Confirm Sync": "将会更新至最新的模板配置，不存在模板中的字段将会被删除（包括所有自定义字段），建议您先复制一份节点，再更新原来节点的版本。",
-    "Custom Title Tip": "该标题名字会展示在对话过程中",
-    "No Modules": "没找到插件"
-  },
-  "modules": {
-    "Title is required": "模块名不能为空"
-  },
  "template": {
    "simple_robot": "简易机器人"
  },
+  "transition_to_workflow": "转成工作流",
+  "transition_to_workflow_create_new_placeholder": "创建一个新的应用，而不是修改当前应用",
+  "transition_to_workflow_create_new_tip": "转化成工作流后，将无法转化回简易模式，请确认！",
  "type": {
    "All": "全部",
    "Create http plugin tip": "通过 OpenAPI Schema 批量创建插件，兼容 GPTs 格式。",

--- a/packages/web/i18n/zh/common.json
+++ b/packages/web/i18n/zh/common.json
--- a/packages/web/i18n/zh/dataset.json
+++ b/packages/web/i18n/zh/dataset.json
@@ -12,6 +12,7 @@
  "external_read_url": "外部预览地址",
  "external_read_url_tip": "可以配置你文件库的阅读地址。便于对用户进行阅读鉴权操作。目前可以使用 {{fileId}} 变量来指代外部文件 ID。",
  "external_url": "文件访问 URL",
+  "folder_dataset": "文件夹",
  "rebuild_embedding_start_tip": "切换索引模型任务已开始",
  "rebuilding_index_count": "重建中索引数量：{{count}}",
  "the_knowledge_base_has_indexes_that_are_being_trained_or_being_rebuilt": "知识库有训练中或正在重建的索引",

--- a/projects/app/src/pages/app/detail/components/WorkflowComponents/Flow/nodes/render/RenderToolInput/index.tsx
+++ b/projects/app/src/pages/app/detail/components/WorkflowComponents/Flow/nodes/render/RenderToolInput/index.tsx
@@ -45,7 +45,7 @@ const RenderToolInput = ({
  return (
    <>
      <HStack mb={2} justifyContent={'space-between'}>
-        <IOTitle text={t('common:core.module.tool.Tool input')} mb={0} />
+        <IOTitle text={t('workflow:tool_input')} mb={0} />
        {dynamicInput && (
          <Button
            variant={'whiteBase'}

--- a/scripts/i18n/delete-unused-keys.js
+++ b/scripts/i18n/delete-unused-keys.js
+const fs = require('fs').promises
+const path = require('path')
+// 配置项
+const CONFIG = {
+  i18nDirectory: path.join(__dirname, '../../packages/web/i18n'),
+  sourceDirectories: ['../../packages', '../../projects/app'].map(dir => path.join(__dirname, dir)),
+  fileExtensions: ['.ts', '.tsx', '.js', '.jsx'],
+  ignoreDirectories: ['node_modules', '.next', 'public', 'i18n']
+}
+// 从文件中加载 JSON
+const loadJSON = async (filePath) => {
+  try {
+    const data = await fs.readFile(filePath, 'utf8')
+    return JSON.parse(data)
+  } catch (error) {
+    console.error(`读取 JSON 文件 ${filePath} 时出错:`, error)
+    return null
+  }
+}
+// 递归提取 JSON 对象的所有键
+const extractKeysFromJSON = (obj, parentKey = '') => {
+  let keys = []
+  for (const [key, value] of Object.entries(obj)) {
+    const fullKey = parentKey ? `${parentKey}.${key}` : key
+    if (typeof value === 'object' && value !== null) {
+      keys = keys.concat(extractKeysFromJSON(value, fullKey))
+    } else {
+      keys.push(fullKey)
+    }
+  }
+  return keys
+}
+// 递归获取文件夹中的所有 JSON 文件
+const getAllJSONFiles = async (dir) => {
+  let results = []
+  const list = await fs.readdir(dir)
+  await Promise.all(list.map(async (file) => {
+    const filePath = path.join(dir, file)
+    const stat = await fs.stat(filePath)
+    if (stat.isDirectory()) {
+      if (!CONFIG.ignoreDirectories.includes(file)) {
+        results = results.concat(await getAllJSONFiles(filePath))
+      }
+    } else if (filePath.endsWith('.json')) {
+      results.push(filePath)
+    }
+  }))
+  return results
+}
+// 提取文件夹中所有 JSON 文件的键
+const extractKeysFromDirectory = async (dir) => {
+  let allKeys = new Set()
+  const subDirs = await fs.readdir(dir)
+  await Promise.all(subDirs.map(async (subDir) => {
+    const subDirPath = path.join(dir, subDir)
+    const stat = await fs.stat(subDirPath)
+    if (stat.isDirectory() && !CONFIG.ignoreDirectories.includes(subDir)) {
+      const files = await getAllJSONFiles(subDirPath)
+      await Promise.all(files.map(async (file) => {
+        const jsonObject = await loadJSON(file)
+        if (jsonObject) {
+          const keys = extractKeysFromJSON(jsonObject)
+          keys.forEach(key => allKeys.add(key))
+        }
+      }))
+    }
+  }))
+  return Array.from(allKeys)
+}
+// 检查键是否在内容中使用
+const isKeyUsedInContent = (content, key) => {
+  const regex = new RegExp(`\\b${key}\\b`, 'g')
+  return regex.test(content)
+}
+// 递归在文件夹中搜索键
+const searchKeysInFiles = async (dir, keys, usedKeys) => {
+  const list = await fs.readdir(dir)
+  await Promise.all(list.map(async (file) => {
+    const filePath = path.join(dir, file)
+    const stat = await fs.stat(filePath)
+    if (stat.isDirectory()) {
+      if (!CONFIG.ignoreDirectories.includes(file)) {
+        await searchKeysInFiles(filePath, keys, usedKeys)
+      }
+    } else if (CONFIG.fileExtensions.includes(path.extname(file))) {
+      const data = await fs.readFile(filePath, 'utf8')
+      keys.forEach(key => {
+        if (isKeyUsedInContent(data, key)) {
+          usedKeys.add(key)
+        }
+      })
+    }
+  }))
+}
+// 从扁平键路径恢复嵌套对象
+const restoreNestedStructure = (keys) => {
+  const result = {}
+  keys.forEach(key => {
+    const parts = key.split('.')
+    let current = result
+    parts.forEach((part, index) => {
+      if (!current[part]) {
+        current[part] = index === parts.length - 1 ? null : {}
+      }
+      current = current[part]
+    })
+  })
+  return result
+}
+// 递归删除未使用的键
+const removeUnusedKeys = (target, unusedKeys) => {
+  for (const [key, value] of Object.entries(unusedKeys)) {
+    if (value === null) {
+      delete target[key]
+    } else if (typeof target[key] === 'object' && target[key] !== null) {
+      removeUnusedKeys(target[key], value)
+      if (Object.keys(target[key]).length === 0) {
+        delete target[key]
+      }
+    }
+  }
+}
+// 处理 JSON 文件
+const processJSONFile = async (filePath, unusedKeys) => {
+  try {
+    const jsonObject = await loadJSON(filePath)
+    if (jsonObject) {
+      removeUnusedKeys(jsonObject, unusedKeys)
+      await fs.writeFile(filePath, JSON.stringify(jsonObject, null, 2), 'utf8')
+      console.log(`已处理文件 ${filePath}`)
+    }
+  } catch (error) {
+    console.error(`处理文件 ${filePath} 时出错:`, error)
+  }
+}
+// 递归处理目录中的所有 JSON 文件
+const processJSONFilesInDirectory = async (dir, unusedKeys) => {
+  const list = await fs.readdir(dir)
+  await Promise.all(list.map(async (file) => {
+    const filePath = path.join(dir, file)
+    const stat = await fs.stat(filePath)
+    if (stat.isDirectory() && !CONFIG.ignoreDirectories.includes(file)) {
+      await processJSONFilesInDirectory(filePath, unusedKeys)
+    } else if (stat.isFile() && file.endsWith('.json')) {
+      await processJSONFile(filePath, unusedKeys)
+    }
+  }))
+}
+// 将 keys 写入 JSON 文件
+const writeKeysToFile = async (filePath, keys) => {
+  try {
+    await fs.writeFile(filePath, JSON.stringify(keys, null, 2), 'utf8')
+    console.log(`已将 keys 写入文件 ${filePath}`)
+  } catch (error) {
+    console.error(`写入文件 ${filePath} 时出错:`, error)
+  }
+}
+// 主函数
+const main = async () => {
+  const allKeys = await extractKeysFromDirectory(CONFIG.i18nDirectory)
+  // await writeKeysToFile(path.join(__dirname, 'allKeys.json'), allKeys)
+  const usedKeys = new Set()
+  await Promise.all(CONFIG.sourceDirectories.map(dir => searchKeysInFiles(dir, allKeys, usedKeys)))
+  const unusedKeys = allKeys.filter(key => !usedKeys.has(key))
+  // await writeKeysToFile(path.join(__dirname, 'unusedKeys.json'), unusedKeys)
+  console.log(unusedKeys)
+  const nestedUnusedKeys = restoreNestedStructure(unusedKeys)
+  await processJSONFilesInDirectory(CONFIG.i18nDirectory, nestedUnusedKeys)
+}
+main().catch(err => console.error('执行过程中出错:', err))
\ No newline at end of file