Skip to content

赵月辉 / fastgpt-migrated

Go to a project
Commit 6c37776d by dreamer6680 Committed by GitHub
Options

fix: resole crawl cannot get docs (#5344)

parent 061547a9
Showing with 170 additions and 261 deletions
document/app/[lang]/robots.txt/route.ts
import * as fs from 'node:fs/promises'; import * as fs from 'node:fs/promises';
import * as path from 'node:path';
import fg from 'fast-glob'; import fg from 'fast-glob';
import matter from 'gray-matter'; import matter from 'gray-matter';
import { i18n } from '@/lib/i18n'; import { i18n } from '@/lib/i18n';
export const revalidate = false; export const revalidate = false;
// 将文件路径转换为URL路径 // 黑名单路径（不带语言前缀）
const blacklist = ['use-cases/index', 'protocol/index', 'api/index'];
// 将文件路径转换为 URL 路径（包括文件名）
function filePathToUrl(filePath: string, defaultLanguage: string): string { function filePathToUrl(filePath: string, defaultLanguage: string): string {
// 移除 ./content/docs/ 前缀 let relativePath = filePath.replace('./content/docs/', '');
let urlPath = filePath.replace('./content/docs/', '');
// 确定基础路径
const basePath = defaultLanguage === 'zh-CN' ? '/docs' : '/en/docs'; const basePath = defaultLanguage === 'zh-CN' ? '/docs' : '/en/docs';
// 如果是英文文件，移除 .en 后缀 if (defaultLanguage !== 'zh-CN' && relativePath.endsWith('.en.mdx')) {
if (defaultLanguage !== 'zh-CN' && urlPath.endsWith('.en.mdx')) { relativePath = relativePath.replace(/\.en\.mdx$/, '');
urlPath = urlPath.replace('.en.mdx', ''); } else if (relativePath.endsWith('.mdx')) {
} else if (urlPath.endsWith('.mdx')) { relativePath = relativePath.replace(/\.mdx$/, '');
urlPath = urlPath.replace('.mdx', '');
}
// 处理 index 文件
if (urlPath.endsWith('/index')) {
urlPath = urlPath.replace('/index', '');
} }
// 拼接完整路径 return `${basePath}/${relativePath}`.replace(/\/\/+/g, '/');
return `${basePath}/${urlPath}`.replace(/\/\/+/g, '/'); }
// 判断是否为黑名单路径
function isBlacklisted(url: string): boolean {
return blacklist.some(
(item) => url.endsWith(`/docs/${item}`) || url.endsWith(`/en/docs/${item}`)
);
} }
export async function GET(request: Request) { export async function GET(request: Request) {
const defaultLanguage = i18n.defaultLanguage; const defaultLanguage = i18n.defaultLanguage;
// 检查请求路径是否为 /en/robots
const requestUrl = new URL(request.url); const requestUrl = new URL(request.url);
const isEnRobotsRoute = requestUrl.pathname === '/en/robots'; const isEnRobotsRoute = requestUrl.pathname === '/en/robots';
let globPattern; let globPattern;
if (isEnRobotsRoute) { if (isEnRobotsRoute) {
// 如果是 /en/robots 路由，只选择 .en.mdx 文件
globPattern = ['./content/docs/**/*.en.mdx']; globPattern = ['./content/docs/**/*.en.mdx'];
} else if (defaultLanguage === 'zh-CN') { } else if (defaultLanguage === 'zh-CN') {
// 中文环境下的普通路由
globPattern = ['./content/docs/**/*.mdx']; globPattern = ['./content/docs/**/*.mdx'];
} else { } else {
// 英文环境下的普通路由
globPattern = ['./content/docs/**/*.en.mdx']; globPattern = ['./content/docs/**/*.en.mdx'];
} }
const files = await fg(globPattern); const files = await fg(globPattern, { caseSensitiveMatch: true });
const urls = await Promise.all( // 转换文件路径为 URL，并过滤黑名单
files.map(async (file: string) => { const urls = files
const urlPath = filePathToUrl(file, defaultLanguage); .map((file) => filePathToUrl(file, defaultLanguage))
return `${urlPath}`; .filter((url) => !isBlacklisted(url));
})
);
// 按URL排序
urls.sort((a, b) => a.localeCompare(b)); urls.sort((a, b) => a.localeCompare(b));
// 生成HTML链接列表
const html = ` const html = `
<html> <html>
<head> <head>
<title>FastGPT Documentation Links</title> <title>FastGPT 文档目录</title>
<style> <style>
body { font-family: Arial, sans-serif; margin: 20px; } body { font-family: Arial, sans-serif; margin: 20px; }
h1 { color: #333; } h1 { color: #333; }
...@@ -78,7 +71,7 @@ export async function GET(request: Request) { ...@@ -78,7 +71,7 @@ export async function GET(request: Request) {
<body> <body>
<h1>Documentation Links</h1> <h1>Documentation Links</h1>
<ul> <ul>
${urls.map(url => `<li><a href="${url}">${url}</a></li>`).join('')} ${urls.map((url) => `<li><a href="${url}">${url}</a></li>`).join('')}
</ul> </ul>
</body> </body>
</html> </html>
...@@ -86,7 +79,7 @@ export async function GET(request: Request) { ...@@ -86,7 +79,7 @@ export async function GET(request: Request) {
return new Response(html, { return new Response(html, {
headers: { headers: {
'Content-Type': 'text/html', 'Content-Type': 'text/html'
}, }
}); });
} }
\ No newline at end of file
document/app/api/meta/route.ts 0 → 100644
import type { NextRequest } from 'next/server';
import { NextResponse } from 'next/server';
import fs from 'fs/promises';
import path from 'path';
const docsRoot = path.resolve(process.cwd(), 'content/docs');
function isInvalidPage(str: string): boolean {
if (!str || typeof str !== 'string') return true;
if (/\[.*?\]\(.*?\)/.test(str) || /^https?:\/\//.test(str) || /[()]/.test(str)) return true;
if (/^\s*---[\s\S]*---\s*$/.test(str)) return true;
return false;
}
function getPageName(str: string): string {
return str.startsWith('...') ? str.slice(3) : str;
}
async function findFirstValidPage(dirRelPath: string): Promise<string | null> {
const absDir = path.join(docsRoot, dirRelPath);
const metaPath = path.join(absDir, 'meta.json');
try {
const metaRaw = await fs.readFile(metaPath, 'utf-8');
const meta = JSON.parse(metaRaw);
if (!Array.isArray(meta.pages)) return null;
for (const page of meta.pages) {
if (isInvalidPage(page)) continue;
const pageName = getPageName(page);
const pagePath = path.join(dirRelPath, pageName);
const candidateDir = path.join(docsRoot, pagePath);
const candidateFile = candidateDir + '.mdx';
try {
await fs.access(candidateFile);
return pagePath;
} catch {
try {
const stat = await fs.stat(candidateDir);
if (stat.isDirectory()) {
const recursiveResult = await findFirstValidPage(pagePath);
if (recursiveResult) return recursiveResult;
}
} catch {
// ignore
}
}
}
} catch {
// ignore
}
return null;
}
export async function GET(req: NextRequest) {
const url = new URL(req.url);
const rawPath = url.searchParams.get('path');
if (!rawPath || !rawPath.startsWith('/docs')) {
return NextResponse.json({ error: 'Invalid path' }, { status: 400 });
}
// 去除 /docs 前缀，且清理首尾斜杠
const relPath = rawPath.replace(/^\/docs\/?/, '').replace(/^\/|\/$/g, '');
try {
// 先检测是否有该 mdx 文件
const maybeFile = path.join(docsRoot, relPath + '.mdx');
await fs.access(maybeFile);
// 如果存在，返回完整路径（带 /docs）
return NextResponse.json('/docs/' + relPath);
} catch {
// 不存在，尝试递归寻找第一个有效页面
const found = await findFirstValidPage(relPath);
if (found) {
// 返回带 /docs 前缀的完整路径
return NextResponse.json('/docs/' + found.replace(/\\/g, '/'));
} else {
return NextResponse.json({ error: 'No valid mdx page found' }, { status: 404 });
}
}
}
document/components/docs/not-found.tsx
'use client'; 'use client';
import { redirect } from 'next/navigation';
import { usePathname } from 'next/navigation';
import { useEffect } from 'react'; import { useEffect } from 'react';
import { usePathname, useRouter } from 'next/navigation';
const exactMap: Record<string, string> = { const exactMap: Record<string, string> = {
'/docs/intro': '/docs/introduction', '/docs/intro': '/docs/introduction',
...@@ -21,25 +20,50 @@ const prefixMap: Record<string, string> = { ...@@ -21,25 +20,50 @@ const prefixMap: Record<string, string> = {
'/docs/agreement': '/docs/protocol' '/docs/agreement': '/docs/protocol'
}; };
const fallbackRedirect = '/docs/introduction';
export default function NotFound() { export default function NotFound() {
const pathname = usePathname(); const pathname = usePathname();
const router = useRouter();
useEffect(() => { useEffect(() => {
if (exactMap[pathname]) { const tryRedirect = async () => {
redirect(exactMap[pathname]); if (exactMap[pathname]) {
return; router.replace(exactMap[pathname]);
}
for (const [oldPrefix, newPrefix] of Object.entries(prefixMap)) {
if (pathname.startsWith(oldPrefix)) {
const rest = pathname.slice(oldPrefix.length);
redirect(newPrefix + rest);
return; return;
} }
}
redirect('/docs/introduction'); for (const [oldPrefix, newPrefix] of Object.entries(prefixMap)) {
}, [pathname]); if (pathname.startsWith(oldPrefix)) {
const rest = pathname.slice(oldPrefix.length);
router.replace(newPrefix + rest);
return;
}
}
try {
const basePath = pathname.replace(/\/$/, '');
const res = await fetch(`/api/meta?path=${basePath}`);
console.log('res', res);
if (!res.ok) throw new Error('meta API not found');
const validPage = await res.json();
if (validPage) {
console.log('validPage', validPage);
router.replace(validPage);
return;
}
} catch (e) {
console.warn('meta.json fallback failed:', e);
}
router.replace(fallbackRedirect);
};
tryRedirect();
}, [pathname, router]);
return <></>; return null;
} }
document/content/docs/api/index.mdx
--- ---
title: API手册 title: API 文档
description: FastGPT API手册 description: API 文档
--- ---
import { Redirect } from '@/components/docs/Redirect'; import { Redirect } from '@/components/docs/Redirect';
......
document/content/docs/index.mdx deleted 100644 → 0
---
title: FastGPT 文档
description: FastGPT 官方文档
---
import { Redirect } from '@/components/docs/Redirect';
<Redirect to="/docs/introduction" />
\ No newline at end of file
document/content/docs/introduction/FAQ/index.mdx deleted 100644 → 0
---
title: FAQ
description: FastGPT FAQ
---
import { Redirect } from '@/components/docs/Redirect';
<Redirect to="/docs/introduction/FAQ/docker" />
\ No newline at end of file
document/content/docs/introduction/development/custom-models/index.mdx deleted 100644 → 0
---
title: 自定义模型
description: FastGPT 自定义模型
---
import { Redirect } from '@/components/docs/Redirect';
<Redirect to="/docs/introduction/development/custom-models/marker" />
\ No newline at end of file
document/content/docs/introduction/development/design/index.mdx deleted 100644 → 0
---
title: 设计文档
description: FastGPT 设计文档
---
import { Redirect } from '@/components/docs/Redirect';
<Redirect to="/docs/introduction/development/design/dataset" />
\ No newline at end of file
document/content/docs/introduction/development/index.mdx deleted 100644 → 0
---
title: 开发文档
description: FastGPT 开发文档
---
import { Redirect } from '@/components/docs/Redirect';
<Redirect to="/docs/introduction/development/intro" />
\ No newline at end of file
document/content/docs/introduction/development/migration/index.mdx deleted 100644 → 0
---
title: 迁移
description: FastGPT 迁移
---
import { Redirect } from '@/components/docs/Redirect';
<Redirect to="/docs/introduction/development/migration/docker_db" />
\ No newline at end of file
document/content/docs/introduction/development/modelConfig/index.mdx deleted 100644 → 0
---
title: 模型配置
description: FastGPT 模型配置
---
import { Redirect } from '@/components/docs/Redirect';
<Redirect to="/docs/introduction/development/modelConfig/ai-proxy" />
\ No newline at end of file
document/content/docs/introduction/development/openapi/index.mdx deleted 100644 → 0
---
title: OpenAPI
description: FastGPT OpenAPI
---
import { Redirect } from '@/components/docs/Redirect';
<Redirect to="/docs/introduction/development/openapi/intro" />
\ No newline at end of file
document/content/docs/introduction/development/proxy/index.mdx deleted 100644 → 0
---
title: 代理
description: FastGPT 代理
---
import { Redirect } from '@/components/docs/Redirect';
<Redirect to="/docs/introduction/development/proxy/nginx" />
\ No newline at end of file
document/content/docs/introduction/development/upgrading/index.mdx deleted 100644 → 0
---
title: 版本更新
description: FastGPT 版本更新
---
import { Redirect } from '@/components/docs/Redirect';
<Redirect to="/docs/introduction/development/upgrading/intro" />
\ No newline at end of file
document/content/docs/introduction/guide/DialogBoxes/index.mdx deleted 100644 → 0
---
title: 对话框
description: FastGPT 对话框
---
import { Redirect } from '@/components/docs/Redirect';
<Redirect to="/docs/introduction/guide/DialogBoxes/htmlRendering" />
\ No newline at end of file
document/content/docs/introduction/guide/admin/index.mdx deleted 100644 → 0
---
title: 商业版管理
description: FastGPT 商业版管理
---
import { Redirect } from '@/components/docs/Redirect';
<Redirect to="/docs/introduction/guide/admin/sso" />
\ No newline at end of file
document/content/docs/introduction/guide/course/index.mdx deleted 100644 → 0
---
title: 基础教程
description: FastGPT 基础教程
---
import { Redirect } from '@/components/docs/Redirect';
<Redirect to="/docs/introduction/guide/course/quick-start" />
\ No newline at end of file
document/content/docs/introduction/guide/dashboard/index.mdx deleted 100644 → 0
---
title: 工作台
description: FastGPT 工作台
---
import { Redirect } from '@/components/docs/Redirect';
<Redirect to="/docs/introduction/guide/dashboard/basic-mode" />
\ No newline at end of file
document/content/docs/introduction/guide/dashboard/workflow/index.mdx deleted 100644 → 0
---
title: 工作流
description: FastGPT 工作流
---
import { Redirect } from '@/components/docs/Redirect';
<Redirect to="/docs/introduction/guide/dashboard/workflow/ai_chat" />
\ No newline at end of file
document/content/docs/introduction/guide/index.mdx deleted 100644 → 0
---
title: 使用指南
description: FastGPT 使用指南
---
import { Redirect } from '@/components/docs/Redirect';
<Redirect to="/docs/introduction/guide/course/quick-start" />
\ No newline at end of file
document/content/docs/introduction/guide/knowledge_base/index.mdx deleted 100644 → 0
---
title: 知识库
description: FastGPT 知识库
---
import { Redirect } from '@/components/docs/Redirect';
<Redirect to="/docs/introduction/guide/knowledge_base/RAG" />
\ No newline at end of file
document/content/docs/introduction/guide/meta.json
{ {
"title": "功能介绍", "title": "功能介绍",
"description": "FastGPT 功能介绍", "description": "FastGPT 功能介绍",
"pages": ["course","dashboard","plugins","knowledge_base","team_permissions","DialogBoxes","admin"] "pages": [
} "course",
\ No newline at end of file "dashboard",
"plugins",
"knowledge_base",
"team_permissions",
"DialogBoxes",
"admin"
]
}
document/content/docs/introduction/guide/plugins/index.mdx deleted 100644 → 0
---
title: 系统插件
description: FastGPT 系统插件
---
import { Redirect } from '@/components/docs/Redirect';
<Redirect to="/docs/introduction/guide/plugins/dev_system_tool" />
\ No newline at end of file
document/content/docs/introduction/guide/team_permissions/index.mdx deleted 100644 → 0
---
title: 团队与权限
description: FastGPT 团队与权限
---
import { Redirect } from '@/components/docs/Redirect';
<Redirect to="/docs/introduction/guide/team_permissions/team_roles_permissions" />
\ No newline at end of file
document/content/docs/introduction/shopping_cart/index.mdx deleted 100644 → 0
---
title: 收费说明
description: FastGPT 收费说明
---
import { Redirect } from '@/components/docs/Redirect';
<Redirect to="/docs/introduction/shopping_cart/saas" />
\ No newline at end of file
document/content/docs/protocol/index.mdx
--- ---
title: FastGPT 协议 title: 协议
description: FastGPT 协议 description: FastGPT 协议
--- ---
import { Redirect } from '@/components/docs/Redirect'; import { Redirect } from '@/components/docs/Redirect';
<Redirect to="/docs/protocol/open-source" /> <Redirect to="/docs/protocol/open-source" />
\ No newline at end of file
document/content/docs/use-cases/app-cases/index.mdx deleted 100644 → 0
---
title: 应用搭建案例
description: FastGPT 应用搭建案例
---
import { Redirect } from '@/components/docs/Redirect';
<Redirect to="/docs/use-cases/app-cases/submit_application_template" />
\ No newline at end of file
document/content/docs/use-cases/external-integration/index.mdx deleted 100644 → 0
---
title: 外部调用 FastGPT
description: FastGPT 外部调用
---
import { Redirect } from '@/components/docs/Redirect';
<Redirect to="/docs/use-cases/external-integration/openapi" />
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment