Updated package.json to add js-yaml dependencies and configure husky hooks to…

Updated package.json to add js-yaml dependencies and configure husky hooks to generate documentation; New llms-full.txt  (#4562)

* 更新 package.json 添加 js-yaml 依赖，并配置 husky 钩子以生成文档；新增 llms-full.txt 和 llms.txt 文件以整理文档链接；修正 ollama.md 中的描述。

* Update the document generation logic to extract and parse the pre-metadata to ensure that the generated document contains the title and description information.

* Optimize the document generation logic, extract and parse the pre-metadata, and ensure that the generated document contains the title and description information.

* Updated document generation logic to change the title and description format from the original metadata format to Markdown format to improve document readability and consistency.

docSite/content/zh-cn/docs/development/custom-models/ollama.md

@@ -76,7 +76,7 @@ ollama serve #安装完成并启动服务后，你可以在浏览器中访问 ht
 
 ### Ollama 拉取模型镜像
 
-在安装后 Ollama 后，本地是没有模型镜像的，需要自己去拉取 Ollama 中的模型镜像。命令如下：
+在安装 Ollama 后，本地是没有模型镜像的，需要自己去拉取 Ollama 中的模型镜像。命令如下：
 
 ```bash
 # Docker 部署需要先进容器，命令为： docker exec -it < Ollama 容器名 > /bin/sh

docSite/static/js/doc-generate-llms.js

+const fs = require('fs');
+const path = require('path');
+const yaml = require('js-yaml');
+
+// 项目名称和描述
+const projectName = "FastGPT";
+const projectDescription = "FastGPT 文档";
+
+// 文档目录，使用相对路径
+const docsDir = path.join(__dirname, '../../content/zh-cn/docs');
+// 基础 URL
+const baseUrl = "https://doc.fastgpt.cn/docs/";
+
+// 生成 llms.txt
+let llmsTxtContent = `# ${projectName}\n${projectDescription}\n`;
+
+function getMdInfo(filePath) {
+    try {
+        // 读取文件内容
+        const content = fs.readFileSync(filePath, 'utf8');
+        // 找到前置元数据的起始和结束位置
+        const startIndex = content.indexOf('---');
+        const endIndex = content.indexOf('---', startIndex + 3);
+        if (startIndex!== -1 && endIndex!== -1) {
+            const frontMatterStr = content.slice(startIndex + 3, endIndex).trim();
+            // 使用 yaml 解析前置元数据
+            const frontMatter = yaml.load(frontMatterStr);
+            const title = frontMatter.title;
+            const description = frontMatter.description;
+            return [title, description];
+        }
+    } catch (error) {
+        if (error.code === 'ENOENT') {
+            console.log(`文件 ${filePath} 未找到。`);
+        } else {
+            console.log(`解析 ${filePath} 的前置元数据时出错:`, error.message);
+        }
+    }
+    return [null, null];
+}
+
+// 遍历文档目录
+function walkDir(dir) {
+    const entries = fs.readdirSync(dir, { withFileTypes: true });
+    for (const entry of entries) {
+        const entryPath = path.join(dir, entry.name);
+        if (entry.isDirectory()) {
+            walkDir(entryPath);
+        } else if (entry.name.endsWith('.md')) {
+            const relativePath = path.relative(docsDir, entryPath);
+            const sectionName = path.dirname(relativePath) || 'Home';
+            if (!llmsTxtContent.includes(`## ${sectionName}`)) {
+                llmsTxtContent += `\n## ${sectionName}\n`;
+            }
+            const fullUrl = baseUrl + relativePath.replace(/\\/g, '/').replace('.md', '/');
+            const [title, description] = getMdInfo(entryPath);
+            const finalTitle = title || path.basename(entry.name, '.md');
+            const finalDescription = description || '';
+            llmsTxtContent += `- [${finalTitle}](${fullUrl}) ${finalDescription}\n`;
+        }
+    }
+}
+
+walkDir(docsDir);
+
+// 保存 llms.txt
+const saveDir = path.join(__dirname, '../');
+if (!fs.existsSync(saveDir)) {
+    fs.mkdirSync(saveDir, { recursive: true });
+}
+const llmsTxtSavePath = path.join(saveDir, 'llms.txt');
+fs.writeFileSync(llmsTxtSavePath, llmsTxtContent, 'utf8');
+
+// 生成 llms - full.txt
+let llmsFullTxtContent = '';
+function collectMdContent(dir) {
+    const entries = fs.readdirSync(dir, { withFileTypes: true });
+    for (const entry of entries) {
+        const entryPath = path.join(dir, entry.name);
+        if (entry.isDirectory()) {
+            collectMdContent(entryPath);
+        } else if (entry.name.endsWith('.md')) {
+            const content = fs.readFileSync(entryPath, 'utf8');
+            // 找到前置元数据的起始和结束位置
+            const startIndex = content.indexOf('---');
+            const endIndex = content.indexOf('---', startIndex + 3);
+            if (startIndex!== -1 && endIndex!== -1) {
+                const frontMatterStr = content.slice(startIndex + 3, endIndex).trim();
+                // 使用 yaml 解析前置元数据
+                const frontMatter = yaml.load(frontMatterStr);
+                const title = frontMatter.title || '';
+                const description = frontMatter.description || '';
+                // 提取标题和描述后，删除首部元数据
+                const newContent = content.slice(endIndex + 3).trim();
+                llmsFullTxtContent += `# ${title}\n## ${description}\n\n${newContent}\n\n`;
+            } else {
+                llmsFullTxtContent += content + '\n\n';
+            }
+        }
+    }
+}
+
+collectMdContent(docsDir);
+
+// 保存 llms - full.txt
+const llmsFullTxtSavePath = path.join(saveDir, 'llms-full.txt');
+fs.writeFileSync(llmsFullTxtSavePath, llmsFullTxtContent, 'utf8');
+    
\ No newline at end of file

package.json

@@ -25,6 +25,7 @@
     "prettier": "3.2.4",
     "react-i18next": "14.1.2",
     "vitest": "^3.0.9",
+    "js-yaml": "^4.1.0",
     "mongodb-memory-server": "^10.1.4",
     "zhlint": "^0.7.4"
   },
@@ -38,5 +39,10 @@
   "engines": {
     "node": ">=18.16.0",
     "pnpm": ">=9.0.0"
+  },
+  "husky": {
+    "hooks": {
+      "pre-commit": "node docSite/static/js/doc-generate-llms.js"
+    }
   }
 }

pnpm-lock.yaml

@@ -23,6 +23,9 @@ importers:
       i18next:
         specifier: 23.16.8
         version: 23.16.8
+      js-yaml:
+        specifier: ^4.1.0
+        version: 4.1.0
       lint-staged:
         specifier: ^13.3.0
         version: 13.3.0