Commit e02e1545 by YeYuheng Committed by GitHub

fix(ai): avoid stack overflow saving base64 request records (#7280)

parent b3200482
...@@ -8,21 +8,33 @@ export const createLLMRequestId = () => { ...@@ -8,21 +8,33 @@ export const createLLMRequestId = () => {
}; };
const base64OmittedPlaceholder = '[base64 omitted]'; const base64OmittedPlaceholder = '[base64 omitted]';
const dataUrlBase64Regex = /data:([^,]*;base64,)([A-Za-z0-9+/=_-]{32,})/g;
const fullBase64Regex = /^[A-Za-z0-9+/=_-]+$/;
const base64LikeKeySet = new Set(['base64', 'data']); const base64LikeKeySet = new Set(['base64', 'data']);
const fullBase64Regex = /^[A-Za-z0-9+/=_-]+$/;
const dataUrlPrefix = 'data:';
const base64UrlMarker = ';base64,';
const rawBase64OmitLength = 256;
const isBase64LikeString = (value: string) => { const isBase64LikeString = (value: string) => {
const text = value.trim(); const text = value.trim();
return text.length >= 256 && fullBase64Regex.test(text); return text.length >= rawBase64OmitLength && fullBase64Regex.test(text);
};
/**
* 清洗整个字段值形式的 data URL。
* 大视频会生成超长 data URL,使用正则 replace 容易触发 V8 栈溢出。
*/
const sanitizeDataUrlBase64 = (value: string) => {
if (!value.startsWith(dataUrlPrefix)) return value;
const markerIndex = value.indexOf(base64UrlMarker);
if (markerIndex === -1) return value;
return `${value.slice(0, markerIndex + base64UrlMarker.length)}${base64OmittedPlaceholder}`;
}; };
const sanitizeString = (value: string, key?: string) => { const sanitizeString = (value: string, key?: string) => {
const sanitizedDataUrl = value.replace( const sanitizedDataUrl = sanitizeDataUrlBase64(value);
dataUrlBase64Regex,
(_match, prefix: string) => `data:${prefix}${base64OmittedPlaceholder}`
);
if (base64LikeKeySet.has(key?.toLowerCase() ?? '') && isBase64LikeString(sanitizedDataUrl)) { if (base64LikeKeySet.has(key?.toLowerCase() ?? '') && isBase64LikeString(sanitizedDataUrl)) {
return base64OmittedPlaceholder; return base64OmittedPlaceholder;
......
...@@ -72,19 +72,42 @@ describe('sanitizeLLMRequestRecordPayload', () => { ...@@ -72,19 +72,42 @@ describe('sanitizeLLMRequestRecordPayload', () => {
text: 'plain text should stay' text: 'plain text should stay'
}); });
}); });
});
it('redacts base64 embedded in a longer string', () => { describe('LLM request record team isolation', () => {
const payload = { it('saves records with very large video data urls after redacting base64 payloads', async () => {
markdown: `![image](data:image/jpeg;base64,${createBase64()})` const largeVideoBase64 = createBase64(10_000_000);
};
expect(sanitizeLLMRequestRecordPayload(payload)).toEqual({ await saveLLMRequestRecord({
markdown: '![image](data:image/jpeg;base64,[base64 omitted])' teamId: '507f1f77bcf86cd799439011',
requestId: 'large_video_request',
body: {
messages: [
{
role: 'user',
content: [
{ type: 'text', text: 'analyze this video' },
{
type: 'video_url',
video_url: {
url: `data:video/mp4;base64,${largeVideoBase64}`
}
}
]
}
]
},
response: { answerText: 'done' }
}); });
const record = await getLLMRequestRecord('large_video_request', '507f1f77bcf86cd799439011');
expect(record?.body.messages[0].content[1].video_url.url).toBe(
'data:video/mp4;base64,[base64 omitted]'
);
expect(record?.response).toEqual({ answerText: 'done' });
}); });
});
describe('LLM request record team isolation', () => {
it('saves records with teamId and only reads them from the same team', async () => { it('saves records with teamId and only reads them from the same team', async () => {
await saveLLMRequestRecord({ await saveLLMRequestRecord({
teamId: '507f1f77bcf86cd799439011', teamId: '507f1f77bcf86cd799439011',
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment