Commit 7b382e80 by ai123 Committed by GitHub

当 DOCX 文件名包含空格时(如 新建 DOCX 文档 (2).docx),会触发以下连锁反应导致图片 404 (#7432)

* Fix regex for S3 markdown and improve VLM model fallback

Update regex pattern for S3 markdown key and adjust fallback logic for VLM model retrieval.

[^)]+? → [^)>]+?:在排除列表中增加 > 字符
(${pattern}) → <?(${pattern})>?:URL 前后添加可选的 <> 包裹匹配

* Implement space replacement in S3 key handling

Add function to replace spaces with underscores in S3 key.

在 replaceParentheses 后新增 replaceSpaces,将所有空白字符(\s+)替换为下划线,避免新上传的文件产生含空格的 S3 key。

* Fix filename formatting tests in utils.test.ts

Updated test cases to ensure formatted filenames replace spaces with underscores and handle multiple spaces correctly.

更新 getFormatedFilename 的空格文件名测试期望
更新真实场景的测试期望
新增 5 个空格替换测试用例(单独替换、括号+空格组合、连续空格、首尾空格等场景)

* fix: handle S3 markdown key edge cases

---------

Co-authored-by: Xianquan <whoeverimf5@gmail.com>
parent 5f7d76f4
...@@ -300,6 +300,10 @@ export function sanitizeS3ObjectKey(key: string) { ...@@ -300,6 +300,10 @@ export function sanitizeS3ObjectKey(key: string) {
}; };
key = replaceParentheses(key); key = replaceParentheses(key);
// 替换空格为下划线,避免 turndown 将含空格的 URL 包裹在 <> 中,
// 导致 replaceS3KeyToPreviewUrl 正则无法匹配图片 key
const replaceSpaces = (key: string) => key.replace(/\s/g, '_');
key = replaceSpaces(key);
return key; return key;
} }
...@@ -12,12 +12,20 @@ import { createS3DownloadAccessUrls } from '../../common/s3/accessLink'; ...@@ -12,12 +12,20 @@ import { createS3DownloadAccessUrls } from '../../common/s3/accessLink';
const logger = getLogger(LogCategories.MODULE.DATASET.FILE); const logger = getLogger(LogCategories.MODULE.DATASET.FILE);
const previewUrlS3Sources = ['dataset', 'chat', 'temp'] as const; const previewUrlS3Sources = ['dataset', 'chat', 'temp'] as const;
/**
* 匹配 Markdown 链接中的 S3 key,同时兼容 Turndown 的 `<...>` 包装。
*
* 尖括号包装与普通 key 分支必须分开匹配,避免把合法 key 中的 `>` 误判为包装结束符。
*/
const createS3MarkdownKeyRegex = () => { const createS3MarkdownKeyRegex = () => {
const pattern = Object.values(S3Sources) const sourcePattern = Object.values(S3Sources)
.map((prefix) => `${prefix}\\/[^)]+?`) .map((prefix) => `${prefix}\\/`)
.join('|'); .join('|');
return new RegExp(String.raw`(!?)\[([^\]]*)\]\(\s*(?!https?:\/\/)(${pattern})\s*\)`, 'g'); return new RegExp(
String.raw`(!?)\[([^\]]*)\]\(\s*(?!https?:\/\/)(?:<((?:${sourcePattern})[^)]+)>|((?:${sourcePattern})[^)]+?))\s*\)`,
'g'
);
}; };
const isPreviewUrlS3ObjectKey = (objectKey: string) => const isPreviewUrlS3ObjectKey = (objectKey: string) =>
...@@ -33,7 +41,7 @@ export const getS3ObjectKeysFromMarkdownTexts = (texts: Array<string | undefined ...@@ -33,7 +41,7 @@ export const getS3ObjectKeysFromMarkdownTexts = (texts: Array<string | undefined
if (!text || typeof text !== 'string') continue; if (!text || typeof text !== 'string') continue;
for (const match of text.matchAll(createS3MarkdownKeyRegex())) { for (const match of text.matchAll(createS3MarkdownKeyRegex())) {
const objectKey = match[3]; const objectKey = match[3] ?? match[4];
if (objectKey && isPreviewUrlS3ObjectKey(objectKey)) { if (objectKey && isPreviewUrlS3ObjectKey(objectKey)) {
objectKeys.add(objectKey); objectKeys.add(objectKey);
} }
...@@ -89,7 +97,8 @@ export const replaceS3KeysWithPreviewUrlMap = ( ...@@ -89,7 +97,8 @@ export const replaceS3KeysWithPreviewUrlMap = (
let content = documentQuoteText; let content = documentQuoteText;
for (const match of matches.slice().reverse()) { for (const match of matches.slice().reverse()) {
const [full, bang, alt, objectKey] = match; const [full, bang, alt, wrappedObjectKey, unwrappedObjectKey] = match;
const objectKey = wrappedObjectKey ?? unwrappedObjectKey;
const previewUrl = objectKey ? previewUrlMap.get(objectKey) : undefined; const previewUrl = objectKey ? previewUrlMap.get(objectKey) : undefined;
if (previewUrl) { if (previewUrl) {
......
...@@ -158,7 +158,7 @@ describe('getFormatedFilename', () => { ...@@ -158,7 +158,7 @@ describe('getFormatedFilename', () => {
it('should handle filename with spaces', () => { it('should handle filename with spaces', () => {
const result = getFormatedFilename('my document.pdf'); const result = getFormatedFilename('my document.pdf');
expect(result.formatedFilename).toBe('my document_abc123'); expect(result.formatedFilename).toBe('my_document_abc123');
expect(result.extension).toBe('pdf'); expect(result.extension).toBe('pdf');
}); });
}); });
...@@ -320,7 +320,7 @@ describe('getFormatedFilename', () => { ...@@ -320,7 +320,7 @@ describe('getFormatedFilename', () => {
it('should handle typical user upload filename', () => { it('should handle typical user upload filename', () => {
const result = getFormatedFilename('My Document (Final Version).pdf'); const result = getFormatedFilename('My Document (Final Version).pdf');
expect(result.formatedFilename).toBe('My Document [Final Version]_abc123'); expect(result.formatedFilename).toBe('My_Document_[Final_Version]_abc123');
expect(result.extension).toBe('pdf'); expect(result.extension).toBe('pdf');
}); });
...@@ -389,7 +389,7 @@ describe('getFormatedFilename', () => { ...@@ -389,7 +389,7 @@ describe('getFormatedFilename', () => {
describe('sanitizeS3ObjectKey', () => { describe('sanitizeS3ObjectKey', () => {
it('should replace parentheses with square brackets', () => { it('should replace parentheses with square brackets', () => {
expect(sanitizeS3ObjectKey('file(1).txt')).toBe('file[1].txt'); expect(sanitizeS3ObjectKey('file(1).txt')).toBe('file[1].txt');
expect(sanitizeS3ObjectKey('photo (copy).jpg')).toBe('photo [copy].jpg'); expect(sanitizeS3ObjectKey('photo (copy).jpg')).toBe('photo_[copy].jpg');
expect(sanitizeS3ObjectKey('document(v2)(final).pdf')).toBe('document[v2][final].pdf'); expect(sanitizeS3ObjectKey('document(v2)(final).pdf')).toBe('document[v2][final].pdf');
}); });
...@@ -425,10 +425,37 @@ describe('sanitizeS3ObjectKey', () => { ...@@ -425,10 +425,37 @@ describe('sanitizeS3ObjectKey', () => {
it('should handle S3 key paths with parentheses', () => { it('should handle S3 key paths with parentheses', () => {
expect(sanitizeS3ObjectKey('dataset/uploads/file (1).pdf')).toBe( expect(sanitizeS3ObjectKey('dataset/uploads/file (1).pdf')).toBe(
'dataset/uploads/file [1].pdf' 'dataset/uploads/file_[1].pdf'
); );
expect(sanitizeS3ObjectKey('chat/images/photo(copy).jpg')).toBe('chat/images/photo[copy].jpg'); expect(sanitizeS3ObjectKey('chat/images/photo(copy).jpg')).toBe('chat/images/photo[copy].jpg');
}); });
it('should replace spaces with underscores', () => {
expect(sanitizeS3ObjectKey('my file.txt')).toBe('my_file.txt');
expect(sanitizeS3ObjectKey('photo copy.jpg')).toBe('photo_copy.jpg');
expect(sanitizeS3ObjectKey('document final version.pdf')).toBe(
'document__final__version.pdf'
);
});
it('should handle both spaces and parentheses together', () => {
expect(sanitizeS3ObjectKey('my file (1).txt')).toBe('my_file_[1].txt');
expect(sanitizeS3ObjectKey('photo (copy) of me.jpg')).toBe('photo_[copy]_of_me.jpg');
expect(sanitizeS3ObjectKey('report (2024) final version.pdf')).toBe(
'report_[2024]_final_version.pdf'
);
});
it('should handle multiple consecutive spaces', () => {
expect(sanitizeS3ObjectKey('file with many spaces.txt')).toBe(
'file___with___many___spaces.txt'
);
});
it('should handle leading and trailing spaces', () => {
expect(sanitizeS3ObjectKey(' file ')).toBe('__file__');
expect(sanitizeS3ObjectKey(' path/to/file ')).toBe('__path/to/file__');
});
}); });
describe('isS3ObjectKey', () => { describe('isS3ObjectKey', () => {
......
...@@ -116,6 +116,21 @@ describe('replaceS3KeyToPreviewUrl', () => { ...@@ -116,6 +116,21 @@ describe('replaceS3KeyToPreviewUrl', () => {
expect(result).toContain('https://example.com/api/system/file/d/mock-short-link-'); expect(result).toContain('https://example.com/api/system/file/d/mock-short-link-');
expect(result).toMatch(/\[文档\]\(https:\/\/example\.com/); expect(result).toMatch(/\[文档\]\(https:\/\/example\.com/);
}); });
it('应替换 Turndown 使用尖括号包装的含空格 S3 key', async () => {
const objectKey = 'dataset/team1/新建 DOCX 文档 [2]_parsed/image.png';
const result = await replaceS3KeyToPreviewUrl(`![image](<${objectKey}>)`, expiredTime);
expect(result).toContain(`mock-short-link-${objectKey}`);
expect(result).not.toContain('<');
});
it('对象键包含大于号时应正常处理', async () => {
const objectKey = 'dataset/team1/a>b.png';
const result = await replaceS3KeyToPreviewUrl(`![image](<${objectKey}>)`, expiredTime);
expect(result).toContain(`mock-short-link-${objectKey}`);
});
}); });
// 测试 chat 前缀的 S3 链接替换 // 测试 chat 前缀的 S3 链接替换
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment