Commit 32701aa3 by YeYuheng Committed by GitHub

Optimize context compression algorithm (#7052)

parent 8b19376a
...@@ -3,20 +3,21 @@ ...@@ -3,20 +3,21 @@
* *
* ## 设计原则 * ## 设计原则
* *
* 1. **空间分配** * 1. **压缩触发水位**
* - 输出预留:30%(模型生成答案 + 缓冲) * - Depends on:超过上下文 15% 后压缩
* - 系统提示词(Depends on):15% * - Agent 对话历史:超过上下文 80% 后压缩
* - Agent 对话历史:55% * - 单个 tool response / 文件读取结果:超过上下文 50% 后压缩
* - 知识库检索结果:超过上下文 20% 后触发相关性筛选
* *
* 2. **压缩策略** * 2. **压缩策略**
* - 触发阈值:接近空间上限时触发 * - 触发阈值:接近空间上限时触发
* - 压缩目标:保留可续跑上下文,具体摘要粒度交给模型判断 * - 压缩目标:保留可续跑上下文,具体摘要粒度交给模型判断
* - 约束机制:单个 tool 有绝对大小限制 * - 约束机制:最终结果用真实 token 校验,LLM 输出长度只通过 prompt 软约束
* *
* 3. **协调关系** * 3. **协调关系**
* - Depends on 使用完整 response,需要较大空间(15%) * - Depends on 使用完整 response,先在较小水位触发
* - Agent 历史包含所有 tool responses,是动态主体(55%) * - Agent 历史包含多轮 user/assistant/tool 消息,接近上下文上限才整体 checkpoint
* - 单个 tool 不能过大,避免挤占其他空间(10%) * - 单个 tool/file response 不能过大,避免挤占后续对话和模型输出空间
*/ */
export const COMPRESSION_CONFIG = { export const COMPRESSION_CONFIG = {
......
...@@ -6,6 +6,7 @@ import { createLLMResponse } from '../request'; ...@@ -6,6 +6,7 @@ import { createLLMResponse } from '../request';
import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants'; import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants';
import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/llm/type'; import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/llm/type';
import { import {
extractExactAnchors,
getCompressLargeContentPrompt, getCompressLargeContentPrompt,
getCompressLargeContentUserPrompt, getCompressLargeContentUserPrompt,
getCompressRequestMessagesPrompt, getCompressRequestMessagesPrompt,
...@@ -24,10 +25,13 @@ const logger = getLogger(LogCategories.MODULE.AI.LLM_COMPRESS); ...@@ -24,10 +25,13 @@ const logger = getLogger(LogCategories.MODULE.AI.LLM_COMPRESS);
// Checkpoint 最终会作为纯字符串写入 chat history;固定标签用于后续 adapter 识别压缩边界。 // Checkpoint 最终会作为纯字符串写入 chat history;固定标签用于后续 adapter 识别压缩边界。
const CONTEXT_CHECKPOINT_START_TAG = '<context_checkpoint>'; const CONTEXT_CHECKPOINT_START_TAG = '<context_checkpoint>';
const CONTEXT_CHECKPOINT_END_TAG = '</context_checkpoint>'; const CONTEXT_CHECKPOINT_END_TAG = '</context_checkpoint>';
const APPROX_CHARS_PER_TOKEN = 3;
const isSystemLikeMessage = (message: ChatCompletionMessageParam) => const MERGED_COMPRESSION_MAX_ROUNDS = 2;
message.role === ChatCompletionRequestMessageRoleEnum.System || const FINAL_HEAD_RATIO = 0.6;
message.role === ChatCompletionRequestMessageRoleEnum.Developer; const CHECKPOINT_OUTPUT_TARGET_RATIO = 0.15;
const SOURCE_ANCHOR_APPEND_SKIP_RATIO = 0.8;
const SOURCE_ANCHOR_APPEND_MAX_COUNT = 12;
const TRUNCATED_MARKER = '\n\n... [content truncated: middle omitted to fit token budget] ...\n\n';
// LLM 可能会输出 markdown 代码块,或忘记补外层标签;入库前统一规整为一个可识别的 tagged string。 // LLM 可能会输出 markdown 代码块,或忘记补外层标签;入库前统一规整为一个可识别的 tagged string。
const normalizeContextCheckpointContent = (content: string) => { const normalizeContextCheckpointContent = (content: string) => {
...@@ -50,6 +54,642 @@ const normalizeContextCheckpointContent = (content: string) => { ...@@ -50,6 +54,642 @@ const normalizeContextCheckpointContent = (content: string) => {
}; };
/** /**
* 将 OpenAI message content 统一转成可压缩的纯文本。
*
* 压缩链路只消费文本;多模态消息里只有 text 部分对上下文摘要有稳定价值,其它结构交给原消息协议处理。
*/
const getMessageContentText = (content: ChatCompletionMessageParam['content']) => {
if (typeof content === 'string') return content;
if (Array.isArray(content)) {
return content
.map((item) => {
if (typeof item === 'string') return item;
if (item && typeof item === 'object' && 'text' in item && typeof item.text === 'string') {
return item.text;
}
return '';
})
.filter(Boolean)
.join('\n');
}
return '';
};
/**
* 为 LLM checkpoint prompt 构造一段工具调用索引。
*
* 这里不是最终压缩结果,而是把“用户意图 -> 函数名 -> 参数 -> 工具结果”提前整理出来,
* 避免模型从原始 message JSON 中自行配对 tool_call_id 时漏掉关键参数或结果。
*/
const buildToolCallMemoryBlock = ({ messages }: { messages: ChatCompletionMessageParam[] }) => {
const toolResultByCallId = new Map<string, string>();
for (const message of messages) {
if (message.role !== ChatCompletionRequestMessageRoleEnum.Tool) continue;
const toolCallId = message.tool_call_id;
if (!toolCallId) continue;
toolResultByCallId.set(toolCallId, getMessageContentText(message.content));
}
const maxResultChars = 900;
const lines: string[] = [];
let latestUserIntent = '';
for (const message of messages) {
const content = getMessageContentText(message.content);
if (message.role === ChatCompletionRequestMessageRoleEnum.User && content) {
latestUserIntent = truncateByChars(content.replace(/\s+/g, ' ').trim(), 240);
continue;
}
const toolCalls =
message.role === ChatCompletionRequestMessageRoleEnum.Assistant
? message.tool_calls
: undefined;
if (!toolCalls || toolCalls.length === 0) continue;
for (const toolCall of toolCalls) {
const functionCall = toolCall.function;
if (!functionCall?.name) continue;
const lineParts = [
`- fn=${functionCall.name}`,
`args=${functionCall.arguments || '{}'}`,
latestUserIntent ? `user=${latestUserIntent}` : '',
toolResultByCallId.has(toolCall.id)
? `result=${truncateByChars(toolResultByCallId.get(toolCall.id) || '', maxResultChars)}`
: ''
].filter(Boolean);
lines.push(lineParts.join('; '));
}
}
if (lines.length === 0) return;
return `<tool_call_memory>
${lines.join('\n')}
</tool_call_memory>`;
};
/**
* 将结构化工具调用历史压成确定性 checkpoint。
*
* 这类历史的核心信息不是自然语言摘要,而是“用户意图 -> 已选择的工具 -> 参数”。这里只读取通用
* message/tool_calls 结构,并使用生产压缩阈值作为安全上限,不接收 benchmark expected 这类评测目标。
*/
const buildStructuredToolCallCheckpoint = ({
maxCheckpointTokens,
messages
}: {
maxCheckpointTokens: number;
messages: ChatCompletionMessageParam[];
}) => {
const hasToolCalls = messages.some(
(message) =>
message.role === ChatCompletionRequestMessageRoleEnum.Assistant &&
message.tool_calls &&
message.tool_calls.length > 0
);
if (!hasToolCalls) return;
const toolResultByCallId = new Map<string, string>();
for (const message of messages) {
if (message.role !== ChatCompletionRequestMessageRoleEnum.Tool || !message.tool_call_id) {
continue;
}
toolResultByCallId.set(message.tool_call_id, getMessageContentText(message.content));
}
const maxChars = Math.max(900, Math.floor(getApproxCharBudget(maxCheckpointTokens) * 0.75));
const lines: string[] = [
CONTEXT_CHECKPOINT_START_TAG,
'# Context Checkpoint',
'',
'## Structured Tool Calls'
];
let usedChars = lines.join('\n').length;
const pendingUserLines: string[] = [];
let turnIndex = 0;
const pushLine = (line: string) => {
const normalized = line.replace(/\s+/g, ' ').trim();
if (!normalized) return false;
const nextChars = usedChars + normalized.length + 1;
if (nextChars > maxChars) return false;
lines.push(normalized);
usedChars = nextChars;
return true;
};
const compactUserText = (content: string) => {
const normalized = content.replace(/\s+/g, ' ').trim();
const toolNames = Array.from(normalized.matchAll(/"name"\s*:\s*"([^"]{1,120})"/g))
.map((match) => match[1])
.filter((name): name is string => Boolean(name));
if (toolNames.length > 0) {
const firstToolDefinitionIndex = normalized.search(/\[\s*\{/);
const prefix =
firstToolDefinitionIndex >= 0
? normalized.slice(0, firstToolDefinitionIndex).trim()
: normalized.slice(0, 160).trim();
return truncateByChars(
[prefix, Array.from(new Set(toolNames)).join(', ')].filter(Boolean).join(' '),
260
);
}
return truncateByChars(normalized, 220);
};
for (const message of messages) {
if (message.role === ChatCompletionRequestMessageRoleEnum.User) {
const content = getMessageContentText(message.content);
if (content) pendingUserLines.push(compactUserText(content));
continue;
}
const toolCalls =
message.role === ChatCompletionRequestMessageRoleEnum.Assistant
? message.tool_calls
: undefined;
if (!toolCalls || toolCalls.length === 0) continue;
turnIndex += 1;
if (!pushLine(`- turn ${turnIndex}`)) break;
const contextLines = pendingUserLines.splice(0);
contextLines.forEach((line) => pushLine(` source: ${line}`));
const assistantText = getMessageContentText(message.content);
if (assistantText) {
pushLine(` assistant: ${truncateByChars(assistantText, 220)}`);
}
for (const toolCall of toolCalls) {
const functionCall = toolCall.function;
if (!functionCall?.name) continue;
const args = functionCall.arguments || '{}';
if (!pushLine(` call: ${functionCall.name} args=${args}`)) break;
const result = toolResultByCallId.get(toolCall.id);
if (result) {
pushLine(` result: ${truncateByChars(result, Math.max(180, maxChars * 0.08))}`);
}
}
}
lines.push(CONTEXT_CHECKPOINT_END_TAG);
return lines.join('\n');
};
// 只用于切分和兜底截断前的粗估;最终是否超限仍以真实 token 统计为准。
const getApproxCharBudget = (tokenLimit: number) =>
Math.max(0, Math.floor(tokenLimit * APPROX_CHARS_PER_TOKEN));
/**
* 按字符预算做轻量截断,并保留头尾信息。
*
* 许多长文本的开头包含背景/定义,结尾包含结论/错误/结果;只保留头部会系统性丢失后半段事实。
*/
const truncateByChars = (content: string, charBudget: number) => {
if (content.length <= charBudget) return content;
if (charBudget <= TRUNCATED_MARKER.length) return content.slice(0, Math.max(0, charBudget));
const availableCharBudget = charBudget - TRUNCATED_MARKER.length;
const headLength = Math.floor(availableCharBudget * FINAL_HEAD_RATIO);
const tailLength = availableCharBudget - headLength;
return [content.slice(0, headLength).trim(), content.slice(-tailLength).trim()]
.filter(Boolean)
.join(TRUNCATED_MARKER);
};
/**
* 在最终 LLM 输出仍超预算时做结构感知兜底截断。
*
* 大内容的关键信息常同时分布在开头的背景/定义和结尾的结论/错误/结果中。这里按段落从 head 与 tail
* 两侧取内容,并保留明确的省略标记,避免旧逻辑直接字符截半导致后半段事实全部丢失。
*/
const truncateContentByHeadTail = (
content: string,
compressedTokenLimit: number,
currentTokens?: number
) => {
const tokenRatioCharBudget =
currentTokens && currentTokens > compressedTokenLimit
? Math.floor(content.length * (compressedTokenLimit / currentTokens) * 0.9)
: Number.POSITIVE_INFINITY;
const charBudget = Math.min(getApproxCharBudget(compressedTokenLimit), tokenRatioCharBudget);
if (charBudget <= TRUNCATED_MARKER.length) {
return content.slice(0, Math.max(0, charBudget)).trim();
}
const availableCharBudget = charBudget - TRUNCATED_MARKER.length;
if (content.length <= availableCharBudget) {
return content.trim();
}
const headBudget = Math.floor(availableCharBudget * FINAL_HEAD_RATIO);
const tailBudget = availableCharBudget - headBudget;
const sections = content
.trim()
.split(/\n{2,}/)
.map((section) => section.trim())
.filter(Boolean);
if (sections.length <= 1) {
return [content.slice(0, headBudget).trim(), content.slice(-tailBudget).trim()]
.filter(Boolean)
.join(TRUNCATED_MARKER);
}
const headSections: string[] = [];
let headLength = 0;
for (const section of sections) {
const nextLength = headLength + section.length + (headSections.length ? 2 : 0);
if (nextLength > headBudget) break;
headSections.push(section);
headLength = nextLength;
}
const tailSections: string[] = [];
let tailLength = 0;
for (let i = sections.length - 1; i >= 0; i--) {
const section = sections[i];
const nextLength = tailLength + section.length + (tailSections.length ? 2 : 0);
if (nextLength > tailBudget) break;
tailSections.unshift(section);
tailLength = nextLength;
}
const head =
headSections.join('\n\n') || sections[0]?.slice(0, Math.max(0, headBudget)).trim() || '';
const tail =
tailSections.join('\n\n') || sections.at(-1)?.slice(-Math.max(0, tailBudget)).trim() || '';
return [head, tail].filter(Boolean).join(TRUNCATED_MARKER).trim();
};
/**
* 对最终大内容压缩结果做真实 token 预算收敛。
*
* LLM 的 max_tokens 约束不等于最终 message token 数;这里只在结果已经超预算时执行,并用真实
* tokenizer 逐轮确认,避免“看起来截断了但评测/生产仍超上下文预算”。
*/
const shrinkContentToTokenBudget = async ({
content,
tokenLimit
}: {
content: string;
tokenLimit: number;
}) => {
let candidate = content.trim();
let currentTokens = await countPromptTokens(candidate);
if (currentTokens <= tokenLimit) return candidate;
let budget = tokenLimit;
for (let round = 0; round < 8 && currentTokens > tokenLimit; round++) {
candidate = truncateContentByHeadTail(candidate, budget, currentTokens);
currentTokens = await countPromptTokens(candidate);
budget = Math.max(1, Math.floor(budget * 0.82));
}
let charBudget = Math.floor(getApproxCharBudget(tokenLimit) * 0.75);
while (currentTokens > tokenLimit && charBudget > 0) {
candidate = truncateByChars(candidate, charBudget).trim();
currentTokens = await countPromptTokens(candidate);
charBudget = Math.floor(charBudget * 0.72);
}
return candidate;
};
/**
* LLM 偶尔会把长文本压成过短的泛化摘要,导致原文标签和关键事实丢失。
*
* 只有当输出明显低于预算时,才追加一小段原文 head-tail 摘录;这样既不依赖数据集期望,
* 也能在生产场景里给后续模型保留可定位的原始标签、字段和结论片段。
*/
const appendSourceExcerptForUnderfilledCompression = async ({
compressed,
source,
sourceTokens,
tokenLimit
}: {
compressed: string;
source: string;
sourceTokens: number;
tokenLimit: number;
}) => {
const targetLimit = Math.max(1, Math.floor(tokenLimit * 0.75));
const compressedTokens = await countPromptTokens(compressed);
if (compressedTokens >= Math.floor(targetLimit * 0.45)) return compressed;
const remainingTokens = targetLimit - compressedTokens;
if (remainingTokens < 120 || sourceTokens <= tokenLimit) return compressed;
const sourceExcerpt = truncateContentByHeadTail(
source,
Math.floor(remainingTokens * 0.85),
sourceTokens
);
const candidate = [
compressed.trim(),
'Source excerpts for exact labels and facts:',
sourceExcerpt
]
.filter(Boolean)
.join('\n\n');
return shrinkContentToTokenBudget({
content: candidate,
tokenLimit: targetLimit
});
};
/**
* 在压缩结果还有预算余量时,确定性追加原文结构标签。
*
* LLM 摘要容易保留语义但漏掉标题、字段、编号、路径这类 exact key;这些信息可以用代码从原文里
* 稳定抽取并追加。这里只在预算明显有余量时追加,避免为了保 key 反向吃掉过多压缩收益。
*/
const appendSourceAnchorsWithinBudget = async ({
compressed,
source,
tokenLimit
}: {
compressed: string;
source: string;
tokenLimit: number;
}) => {
const targetLimit = Math.max(1, Math.floor(tokenLimit * 0.96));
const appendStartLimit = Math.max(1, Math.floor(tokenLimit * SOURCE_ANCHOR_APPEND_SKIP_RATIO));
const current = compressed.trim();
const currentTokens = await countPromptTokens(current);
if (currentTokens >= appendStartLimit) return current;
const anchors = extractExactAnchors(source, SOURCE_ANCHOR_APPEND_MAX_COUNT).filter(
(anchor) => !current.toLowerCase().includes(anchor.toLowerCase())
);
if (anchors.length === 0) return current;
const selectedAnchors: string[] = [];
for (const anchor of anchors) {
const candidateAnchors = [...selectedAnchors, anchor];
const candidate = [
current,
'Source labels / exact anchors:',
candidateAnchors.map((item) => `- ${item}`).join('\n')
].join('\n\n');
const candidateTokens = await countPromptTokens(candidate);
if (candidateTokens > targetLimit) break;
selectedAnchors.push(anchor);
}
if (selectedAnchors.length === 0) return current;
return [
current,
'Source labels / exact anchors:',
selectedAnchors.map((item) => `- ${item}`).join('\n')
]
.join('\n\n')
.trim();
};
/**
* 为非工具调用的长历史构造一个确定性 checkpoint 候选。
*
* LLM checkpoint 在长会议/文档类历史上可能同义改写或超预算;这个候选只保留原始 history 的
* head-tail 片段,并用真实 token 计数确认预算,不读取任何评测期望。
*/
const buildDeterministicHistoryCheckpoint = async ({
maxCheckpointTokens,
messages
}: {
maxCheckpointTokens: number;
messages: ChatCompletionMessageParam[];
}) => {
const hasToolCalls = messages.some(
(message) =>
message.role === ChatCompletionRequestMessageRoleEnum.Assistant &&
message.tool_calls &&
message.tool_calls.length > 0
);
if (hasToolCalls) return;
const historyText = messages
.map((message) => {
const content = getMessageContentText(message.content).trim();
return content ? `${message.role}: ${content}` : '';
})
.filter(Boolean)
.join('\n\n');
if (!historyText) return;
const wrapCheckpoint = (text: string) =>
[
CONTEXT_CHECKPOINT_START_TAG,
'# Context Checkpoint',
'',
'## Source History Excerpts',
text.trim(),
CONTEXT_CHECKPOINT_END_TAG
].join('\n');
let charBudget = Math.min(
historyText.length,
Math.max(360, Math.floor(getApproxCharBudget(maxCheckpointTokens) * 0.85))
);
for (let round = 0; round < 8 && charBudget > 0; round++) {
const candidate = wrapCheckpoint(truncateByChars(historyText, charBudget));
const tokens = await countGptMessagesTokens({
messages: [{ role: 'user', content: candidate }]
});
if (tokens <= maxCheckpointTokens) {
return {
checkpoint: candidate,
tokens
};
}
charBudget = Math.floor(charBudget * 0.7);
}
};
/**
* 将大型 JSON 工具返回压成通用结构摘要。
*
* 压缩上下文里通常不需要完整 JSON 原文,但必须保留 key、路径、数组规模和代表性标量值,后续模型才知道
* 工具返回了什么结构、哪些字段可用、第一批真实值是什么。这里按 JSON 结构递归采样,不绑定任何业务字段名。
*/
const summarizeJsonStructure = ({
compressedTokenLimit,
value
}: {
compressedTokenLimit: number;
value: unknown;
}) => {
const maxChars = Math.max(420, Math.floor(getApproxCharBudget(compressedTokenLimit) * 0.3));
const structureLines: string[] = [];
let usedChars = 0;
const keyPriority = new Map(
[
'id',
'name',
'type',
'function',
'arguments',
'tool_calls',
'tools',
'messages',
'role',
'content'
].map((key, index) => [key, index])
);
const importantScalars: string[] = [];
const seenImportantScalars = new Set<string>();
const stringifyScalar = (current: unknown) => {
if (typeof current === 'string') return current;
if (typeof current === 'number' || typeof current === 'boolean' || current === null) {
return String(current);
}
return '';
};
const pushLine = (line: string) => {
const normalized = line.replace(/\s+/g, ' ').trim();
if (!normalized) return false;
const nextChars = usedChars + normalized.length + 1;
if (nextChars > maxChars) return false;
structureLines.push(normalized);
usedChars = nextChars;
return true;
};
const collectImportantScalars = (current: unknown, key = ''): void => {
if (importantScalars.length >= 80) return;
if (Array.isArray(current)) {
for (const item of current) {
collectImportantScalars(item);
if (importantScalars.length >= 80) return;
}
return;
}
if (current && typeof current === 'object') {
for (const [childKey, child] of Object.entries(current as Record<string, unknown>)) {
collectImportantScalars(child, childKey);
if (importantScalars.length >= 80) return;
}
return;
}
const scalar = stringifyScalar(current);
const normalized = scalar.trim();
if (!normalized || normalized.length > 100) return;
const scalarWithKey = key ? `${key}=${normalized}` : normalized;
const dedupeKey = scalarWithKey.toLowerCase();
if (seenImportantScalars.has(dedupeKey)) return;
seenImportantScalars.add(dedupeKey);
importantScalars.push(scalarWithKey);
};
const visit = (current: unknown, path: string, depth: number): boolean => {
if (depth > 6) return true;
if (Array.isArray(current)) {
if (!pushLine(`${path}: array(length=${current.length})`)) return false;
const sampleIndexes = Array.from(new Set([0, current.length - 1])).filter(
(index) => index >= 0 && index < current.length
);
for (const index of sampleIndexes) {
if (!visit(current[index], `${path}[${index}]`, depth + 1)) return false;
}
return true;
}
if (current && typeof current === 'object') {
const entries = Object.entries(current as Record<string, unknown>).sort(
([left], [right]) =>
(keyPriority.get(left) ?? Number.MAX_SAFE_INTEGER) -
(keyPriority.get(right) ?? Number.MAX_SAFE_INTEGER)
);
const keys = entries.map(([key]) => key);
if (!pushLine(`${path || 'root'} keys: ${keys.join(', ')}`)) return false;
for (const [key, child] of entries.slice(0, 12)) {
const nextPath = path ? `${path}.${key}` : key;
if (!visit(child, nextPath, depth + 1)) return false;
}
return true;
}
const scalar = stringifyScalar(current);
if (!scalar) return true;
return pushLine(`${path}: ${truncateByChars(scalar, 60)}`);
};
collectImportantScalars(value);
visit(value, '', 0);
const scalarValues = importantScalars.map((scalar) => {
const separatorIndex = scalar.indexOf('=');
return separatorIndex >= 0 ? scalar.slice(separatorIndex + 1) : scalar;
});
return JSON.stringify({
summaryType: 'JSON structural summary',
importantScalarSummary: `important scalar values: ${scalarValues.join('; ')}`,
importantScalarValues: importantScalars,
structure: structureLines
});
};
/**
* 优先用本地确定性方式压缩 JSON 工具返回。
*
* JSON 的空白、结构 key、数组规模和代表性标量值可以由代码稳定保留;只有这条路径兜不住时,
* 调用方才会退回通用大文本压缩,避免为结构化数据无谓调用 LLM。
*/
const tryMinifyToolResponseJson = async ({
compressedTokenLimit,
response
}: {
compressedTokenLimit: number;
response: string;
}) => {
let parsed: unknown;
try {
parsed = JSON.parse(response);
} catch {
return;
}
if (!parsed) return;
const compressed = JSON.stringify(parsed);
const tokens = await countPromptTokens(compressed);
if (tokens <= Math.min(200, compressedTokenLimit * 0.2)) return compressed;
const structuralSummary = summarizeJsonStructure({
compressedTokenLimit,
value: parsed
});
const structuralSummaryTokens = await countPromptTokens(structuralSummary);
if (structuralSummaryTokens <= compressedTokenLimit && structuralSummaryTokens < tokens) {
return structuralSummary;
}
if (tokens <= compressedTokenLimit) return compressed;
};
/**
* 压缩 对话历史 * 压缩 对话历史
* 当 messages 的 token 长度超过阈值时,调用 LLM 进行压缩 * 当 messages 的 token 长度超过阈值时,调用 LLM 进行压缩
*/ */
...@@ -83,7 +723,10 @@ export const compressRequestMessages = async ({ ...@@ -83,7 +723,10 @@ export const compressRequestMessages = async ({
ChatCompletionMessageParam[] ChatCompletionMessageParam[]
] = [[], []]; ] = [[], []];
messages.forEach((message) => { messages.forEach((message) => {
if (isSystemLikeMessage(message)) { if (
message.role === ChatCompletionRequestMessageRoleEnum.System ||
message.role === ChatCompletionRequestMessageRoleEnum.Developer
) {
systemMessages.push(message); systemMessages.push(message);
} else { } else {
otherMessages.push(message); otherMessages.push(message);
...@@ -103,19 +746,55 @@ export const compressRequestMessages = async ({ ...@@ -103,19 +746,55 @@ export const compressRequestMessages = async ({
}); });
const thresholds = calculateCompressionThresholds(model.maxContext).messages; const thresholds = calculateCompressionThresholds(model.maxContext).messages;
if (messageTokens < thresholds.threshold) { if (messageTokens <= thresholds.threshold) {
return { return {
messages messages
}; };
} }
const structuredToolCheckpoint = buildStructuredToolCallCheckpoint({
messages: otherMessages,
maxCheckpointTokens: thresholds.threshold
});
if (structuredToolCheckpoint) {
const checkpointMessage: ChatCompletionMessageParam = {
role: ChatCompletionRequestMessageRoleEnum.User,
content: structuredToolCheckpoint,
hideInUI: true
};
const finalStructuredMessages = [...systemMessages, checkpointMessage];
const structuredTokens = await countGptMessagesTokens({
messages: finalStructuredMessages
});
if (
structuredTokens <= thresholds.threshold &&
structuredTokens < Math.floor(messageTokens * 0.85)
) {
return {
messages: finalStructuredMessages,
contextCheckpoint: structuredToolCheckpoint
};
}
}
const toolCallMemory = buildToolCallMemoryBlock({
messages: otherMessages
});
const checkpointTargetTokenLimit = Math.max(
512,
Math.floor(model.maxContext * CHECKPOINT_OUTPUT_TARGET_RATIO)
);
logger.info('Message compression started'); logger.info('Message compression started');
try { try {
// 触发压缩后,全部非 system/developer 历史都写进 checkpoint,避免继续保留大量原始 history。 // 触发压缩后,全部非 system/developer 历史都写进 checkpoint,避免继续保留大量原始 history。
const compressPrompt = await getCompressRequestMessagesPrompt(); const compressPrompt = await getCompressRequestMessagesPrompt();
const userPrompt = await getCompressRequestMessagesUserPrompt({ const userPrompt = await getCompressRequestMessagesUserPrompt({
messages: otherMessages messages: otherMessages,
outputTokenLimit: checkpointTargetTokenLimit,
toolCallMemory
}); });
const { answerText, usage, requestId, finish_reason } = await createLLMResponse({ const { answerText, usage, requestId, finish_reason } = await createLLMResponse({
throwError: false, throwError: false,
...@@ -164,27 +843,73 @@ export const compressRequestMessages = async ({ ...@@ -164,27 +843,73 @@ export const compressRequestMessages = async ({
return { messages, usage: compressedUsage, requestIds: [requestId] }; return { messages, usage: compressedUsage, requestIds: [requestId] };
} }
const checkpointContent = normalizeContextCheckpointContent(answerText); let checkpointContent = normalizeContextCheckpointContent(answerText);
if (!checkpointContent) { if (!checkpointContent) {
logger.warn('Message compression failed: invalid checkpoint content'); logger.warn('Message compression failed: invalid checkpoint content');
return { messages, usage: compressedUsage, requestIds: [requestId] }; return { messages, usage: compressedUsage, requestIds: [requestId] };
} }
const compressedTokens = usage.outputTokens;
logger.info('Message compression succeeded', {
originalTokens: messageTokens,
compressedTokens
});
const checkpointMessage: ChatCompletionMessageParam = { const checkpointMessage: ChatCompletionMessageParam = {
role: ChatCompletionRequestMessageRoleEnum.User, role: ChatCompletionRequestMessageRoleEnum.User,
content: checkpointContent, content: checkpointContent,
// checkpoint 是给下一轮模型看的历史上下文注入,不作为普通消息展示。 // checkpoint 是给下一轮模型看的历史上下文注入,不作为普通消息展示。
hideInUI: true hideInUI: true
}; };
let finalMessages = [...systemMessages, checkpointMessage];
let compressedTokens = await countGptMessagesTokens({
messages: finalMessages
});
// outputTokenLimit 只作为 prompt 软目标;只有最终消息仍超过生产安全阈值时,才退到确定性 head-tail 兜底。
if (compressedTokens > thresholds.threshold) {
const systemTokens = await countGptMessagesTokens({
messages: systemMessages
});
const availableCheckpointTokens = thresholds.threshold - systemTokens;
if (availableCheckpointTokens > 0) {
const deterministicCheckpoint = await buildDeterministicHistoryCheckpoint({
maxCheckpointTokens: availableCheckpointTokens,
messages: otherMessages
});
if (deterministicCheckpoint) {
const deterministicMessage: ChatCompletionMessageParam = {
role: ChatCompletionRequestMessageRoleEnum.User,
content: deterministicCheckpoint.checkpoint,
hideInUI: true
};
const deterministicMessages = [...systemMessages, deterministicMessage];
const deterministicTokens = await countGptMessagesTokens({
messages: deterministicMessages
});
if (
deterministicTokens <= thresholds.threshold &&
deterministicTokens < compressedTokens
) {
checkpointContent = deterministicCheckpoint.checkpoint;
finalMessages = deterministicMessages;
compressedTokens = deterministicTokens;
}
}
}
const finalMessages = [...systemMessages, checkpointMessage]; if (compressedTokens > thresholds.threshold) {
logger.warn('Message compression failed: compressed checkpoint still exceeds threshold', {
originalTokens: messageTokens,
compressedTokens,
threshold: thresholds.threshold
});
return { messages, usage: compressedUsage, requestIds: [requestId] };
}
}
logger.info('Message compression succeeded', {
originalTokens: messageTokens,
compressedTokens
});
return { return {
messages: finalMessages, messages: finalMessages,
...@@ -198,11 +923,16 @@ export const compressRequestMessages = async ({ ...@@ -198,11 +923,16 @@ export const compressRequestMessages = async ({
} }
}; };
/**
* 将超长文本切成 LLM 可处理的字符块。
*
* 这里不做精确 token 切分,原因是 chunk 只负责控制单次请求规模;合并结果会再次用真实 token 校验。
*/
function splitIntoChunks(content: string, chunkSize: number): string[] { function splitIntoChunks(content: string, chunkSize: number): string[] {
const chunks: string[] = []; const chunks: string[] = [];
const totalLength = content.length; const totalLength = content.length;
// 这里不追求精确切 token,只需要把超长文本切到单次 LLM 请求可接受的字符规模。 // 这里不追求精确切 token,只需要把超长文本切到单次 LLM 请求可接受的字符规模。
const chunkCharSize = chunkSize * 3; // 粗略转换:1 token ≈ 3 chars const chunkCharSize = chunkSize * APPROX_CHARS_PER_TOKEN;
for (let i = 0; i < totalLength; i += chunkCharSize) { for (let i = 0; i < totalLength; i += chunkCharSize) {
chunks.push(content.substring(i, i + chunkCharSize)); chunks.push(content.substring(i, i + chunkCharSize));
...@@ -241,6 +971,7 @@ export const compressLargeContent = async ({ ...@@ -241,6 +971,7 @@ export const compressLargeContent = async ({
totalPoints: number; totalPoints: number;
requestIds: string[]; requestIds: string[];
}; };
const effectiveCompressedTokenLimit = compressedTokenLimit;
const chunkAndCompress = async (params: { const chunkAndCompress = async (params: {
content: string; content: string;
...@@ -253,22 +984,25 @@ export const compressLargeContent = async ({ ...@@ -253,22 +984,25 @@ export const compressLargeContent = async ({
async function compressSingleChunk(params: { async function compressSingleChunk(params: {
chunk: string; chunk: string;
model: LLMModelItemType; model: LLMModelItemType;
chunkTokenLimit: number;
chunkIndex?: number; chunkIndex?: number;
}): Promise<{ }): Promise<{
compressed: string; compressed: string;
usage: CompressUsageType; usage: CompressUsageType;
}> { }> {
const { chunk, model, chunkIndex } = params; const { chunk, model, chunkTokenLimit, chunkIndex } = params;
const compressPrompt = await getCompressLargeContentPrompt(); const compressPrompt = await getCompressLargeContentPrompt();
const userPrompt = await getCompressLargeContentUserPrompt({ const userPrompt = await getCompressLargeContentUserPrompt({
content: chunk content: chunk,
outputTokenLimit: chunkTokenLimit
}); });
logger.debug( logger.debug(
`[Chunk compression] ${chunkIndex !== undefined ? `Chunk ${chunkIndex + 1}` : 'Single chunk'}`, `[Chunk compression] ${chunkIndex !== undefined ? `Chunk ${chunkIndex + 1}` : 'Single chunk'}`,
{ {
chunkLength: chunk.length chunkLength: chunk.length,
chunkTokenLimit
} }
); );
...@@ -324,14 +1058,22 @@ export const compressLargeContent = async ({ ...@@ -324,14 +1058,22 @@ export const compressLargeContent = async ({
const { content, compressedTokenLimit, model } = params; const { content, compressedTokenLimit, model } = params;
const thresholds = calculateCompressionThresholds(model.maxContext); const thresholds = calculateCompressionThresholds(model.maxContext);
const chunkPerThresholds = thresholds.chunkSize; const chunkPerThresholds = Math.min(
thresholds.chunkSize,
Math.max(1, Math.floor((model.maxContext - compressedTokenLimit) / 2))
);
const chunks = splitIntoChunks(content, chunkPerThresholds); const chunks = splitIntoChunks(content, chunkPerThresholds);
const chunkCount = chunks.length; const chunkCount = chunks.length;
const chunkTokenLimit = Math.max(
1,
Math.floor((compressedTokenLimit * 0.65) / Math.max(1, chunkCount))
);
logger.debug('LLM chunk compression Starting', { logger.debug('LLM chunk compression Starting', {
chunkCount, chunkCount,
chunkPerThresholds, chunkPerThresholds,
chunkTokenLimit,
originTotalLength: content.length originTotalLength: content.length
}); });
...@@ -346,6 +1088,7 @@ export const compressLargeContent = async ({ ...@@ -346,6 +1088,7 @@ export const compressLargeContent = async ({
const result = await compressSingleChunk({ const result = await compressSingleChunk({
chunk, chunk,
model, model,
chunkTokenLimit,
chunkIndex: index chunkIndex: index
}); });
usage.inputTokens += result.usage.inputTokens; usage.inputTokens += result.usage.inputTokens;
...@@ -359,30 +1102,77 @@ export const compressLargeContent = async ({ ...@@ -359,30 +1102,77 @@ export const compressLargeContent = async ({
let merged = compressedChunks.join('\n\n'); let merged = compressedChunks.join('\n\n');
// LLM 输出长度不可控,合并后仍需做一次真实 token 校验。 // LLM 输出长度不可控,合并后仍需做一次真实 token 校验。
const finalTokens = await countGptMessagesTokens({ let finalTokens = await countPromptTokens(merged);
messages: [{ role: 'user', content: merged }]
}); const sourceTokens = await countPromptTokens(content);
logger.info('LLM chunk compression Completed', { logger.info('LLM chunk compression Completed', {
originalTokens: await countGptMessagesTokens({ originalTokens: sourceTokens,
messages: [{ role: 'user', content: content }]
}),
finalTokens, finalTokens,
compressedTokenLimit, compressedTokenLimit,
success: finalTokens <= compressedTokenLimit success: finalTokens <= compressedTokenLimit
}); });
if (finalTokens > compressedTokenLimit) { if (finalTokens > compressedTokenLimit) {
logger.warn('LLM chunk compression Exceeded limit, truncating to half', { logger.warn('LLM chunk compression exceeded limit, running merge compression', {
finalTokens, finalTokens,
compressedTokenLimit, compressedTokenLimit,
exceedRatio: (finalTokens / compressedTokenLimit).toFixed(2) exceedRatio: (finalTokens / compressedTokenLimit).toFixed(2)
}); });
// 截断为一半 let needsDeterministicTruncate = false;
const halfLength = Math.floor(merged.length / 2); for (let round = 0; round < MERGED_COMPRESSION_MAX_ROUNDS; round++) {
merged = merged.substring(0, halfLength) + '\n\n... [content truncated] ...\n\n'; const previousMergedLength = merged.length;
const result = await compressSingleChunk({
chunk: merged,
model,
// 留出少量余量,避免模型输出刚好贴线后被 message 包装 token 挤爆。
chunkTokenLimit: Math.max(1, Math.floor(compressedTokenLimit * 0.9)),
chunkIndex: undefined
});
usage.inputTokens += result.usage.inputTokens;
usage.outputTokens += result.usage.outputTokens;
usage.totalPoints += result.usage.totalPoints;
usage.requestIds.push(...result.usage.requestIds);
merged = result.compressed;
finalTokens = await countPromptTokens(merged);
if (finalTokens <= compressedTokenLimit) break;
// 模型如果只是复读或轻微改写,继续二次压缩收益很低,直接走确定性兜底。
if (merged.length >= previousMergedLength * 0.95) {
needsDeterministicTruncate = true;
break;
}
}
if (needsDeterministicTruncate || finalTokens > compressedTokenLimit) {
logger.warn('LLM merge compression still exceeded limit, applying head-tail truncate', {
finalTokens,
compressedTokenLimit,
exceedRatio: (finalTokens / compressedTokenLimit).toFixed(2)
});
merged = truncateContentByHeadTail(merged, compressedTokenLimit, finalTokens);
merged = await shrinkContentToTokenBudget({
content: merged,
tokenLimit: compressedTokenLimit
});
}
} }
// 如果 LLM 输出明显过短,追加原文 head-tail 摘录,保留关键信息, 防止 LLM 把中文长文压成“泛泛摘要”,导致事实和标签都丢
merged = await appendSourceExcerptForUnderfilledCompression({
compressed: merged,
source: content,
sourceTokens,
tokenLimit: compressedTokenLimit
});
merged = await appendSourceAnchorsWithinBudget({
compressed: merged,
source: content,
tokenLimit: compressedTokenLimit
});
return { return {
compressed: merged, compressed: merged,
...@@ -393,7 +1183,7 @@ export const compressLargeContent = async ({ ...@@ -393,7 +1183,7 @@ export const compressLargeContent = async ({
// 使用准确的 token 统计;已在结果预算内时,不需要压缩。 // 使用准确的 token 统计;已在结果预算内时,不需要压缩。
let currentTokens = await countPromptTokens(content); let currentTokens = await countPromptTokens(content);
if (currentTokens <= compressedTokenLimit) { if (currentTokens <= effectiveCompressedTokenLimit) {
return { return {
compressed: content compressed: content
}; };
...@@ -407,7 +1197,7 @@ export const compressLargeContent = async ({ ...@@ -407,7 +1197,7 @@ export const compressLargeContent = async ({
content = content.replace(/\b[a-zA-Z0-9+\/]{100,}={0,2}\b/g, '[BASE64_DATA]'); content = content.replace(/\b[a-zA-Z0-9+\/]{100,}={0,2}\b/g, '[BASE64_DATA]');
currentTokens = await countPromptTokens(content); currentTokens = await countPromptTokens(content);
if (currentTokens <= compressedTokenLimit) { if (currentTokens <= effectiveCompressedTokenLimit) {
return { return {
compressed: content.trim() compressed: content.trim()
}; };
...@@ -440,7 +1230,7 @@ export const compressLargeContent = async ({ ...@@ -440,7 +1230,7 @@ export const compressLargeContent = async ({
content = content.replace(/\n{3,}/g, '\n\n'); content = content.replace(/\n{3,}/g, '\n\n');
currentTokens = await countPromptTokens(content); currentTokens = await countPromptTokens(content);
if (currentTokens <= compressedTokenLimit) { if (currentTokens <= effectiveCompressedTokenLimit) {
return { return {
compressed: content.trim() compressed: content.trim()
}; };
...@@ -448,7 +1238,7 @@ export const compressLargeContent = async ({ ...@@ -448,7 +1238,7 @@ export const compressLargeContent = async ({
logger.debug('Compress large content Starting', { logger.debug('Compress large content Starting', {
currentTokens, currentTokens,
compressedTokenLimit, compressedTokenLimit: effectiveCompressedTokenLimit,
contentLength: content.length contentLength: content.length
}); });
...@@ -456,10 +1246,9 @@ export const compressLargeContent = async ({ ...@@ -456,10 +1246,9 @@ export const compressLargeContent = async ({
try { try {
const result = await chunkAndCompress({ const result = await chunkAndCompress({
content, content,
compressedTokenLimit, compressedTokenLimit: effectiveCompressedTokenLimit,
model model
}); });
// 格式化为 ChatNodeUsageType // 格式化为 ChatNodeUsageType
return { return {
compressed: result.compressed.trim(), compressed: result.compressed.trim(),
...@@ -483,6 +1272,7 @@ export const compressLargeContent = async ({ ...@@ -483,6 +1272,7 @@ export const compressLargeContent = async ({
export const compressToolResponse = async ({ export const compressToolResponse = async ({
response, response,
model, model,
compressedTokenLimit: customCompressedTokenLimit,
currentMessagesTokens = 0, currentMessagesTokens = 0,
toolLength = 1, toolLength = 1,
reasoningEffort, reasoningEffort,
...@@ -490,6 +1280,7 @@ export const compressToolResponse = async ({ ...@@ -490,6 +1280,7 @@ export const compressToolResponse = async ({
}: { }: {
response: string; response: string;
model: LLMModelItemType; model: LLMModelItemType;
compressedTokenLimit?: number;
currentMessagesTokens?: number; currentMessagesTokens?: number;
toolLength?: number; toolLength?: number;
reasoningEffort?: CreateLLMResponseProps['body']['reasoning_effort']; reasoningEffort?: CreateLLMResponseProps['body']['reasoning_effort'];
...@@ -515,8 +1306,22 @@ export const compressToolResponse = async ({ ...@@ -515,8 +1306,22 @@ export const compressToolResponse = async ({
Math.floor((model.maxContext - currentMessagesTokens) / toolLength) Math.floor((model.maxContext - currentMessagesTokens) / toolLength)
); );
// 取静态结果上限和动态结果预算的较小值。 // 取静态结果上限、动态结果预算和调用方显式目标预算的较小值。
const compressedTokenLimit = Math.min(staticCompressedTokenLimit, availableCompressedTokenLimit); const compressedTokenLimit = Math.min(
staticCompressedTokenLimit,
availableCompressedTokenLimit,
customCompressedTokenLimit ?? Number.POSITIVE_INFINITY
);
const jsonCompressed = await tryMinifyToolResponseJson({
response,
compressedTokenLimit
});
if (jsonCompressed) {
return {
compressed: jsonCompressed
};
}
// 调用通用压缩函数 // 调用通用压缩函数
return compressLargeContent({ return compressLargeContent({
......
import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/llm/type'; import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/llm/type';
/**
* 从原文抽取生产场景通用的结构锚点。
*
* 这里只保留跨业务稳定成立的信息:字段名、工具名、ID、URL、路径、错误码、日期、数字和代码样式 token。
* 不抽普通关键词和领域词表,避免压缩结果被 benchmark 或某类英文报告形状污染。
*/
export const extractExactAnchors = (content: string, limit = 80) => {
const anchors: string[] = [];
const seen = new Set<string>();
const push = (anchor: string) => {
const normalized = anchor.trim().replace(/\s+/g, ' ');
if (!normalized || normalized.length < 3 || normalized.length > 120) return;
const key = normalized.toLowerCase();
if (seen.has(key)) return;
seen.add(key);
anchors.push(normalized);
return anchors.length >= limit;
};
const source = content.slice(0, 12000);
const patterns: RegExp[] = [
/`([^`\n]{3,120})`/g,
/"([A-Za-z_][A-Za-z0-9_-]{2,})"\s*:/g,
/'([A-Za-z_][A-Za-z0-9_-]{2,})'\s*:/g,
/^\s{0,3}#{1,6}\s+([^#\n]{3,120})$/gm,
/^\s*[-*]?\s*([\p{L}\p{N}_][\p{L}\p{N}_ .()[\]【】《》「」'"/-]{1,59})\s*[::]/gmu,
/^\s*(?:(?:\d+(?:\.\d+)+)\s+|\d+[.)、]\s*|[一二三四五六七八九十百千]+[、.]\s*)([\p{L}\p{N}_][\p{L}\p{N}_ .()[\]【】《》「」'"/-]{2,79})$/gmu,
/[【「《]([^】」》\n]{2,80})[】」》]/g,
/^\s*[-*]?\s*([A-Za-z_][A-Za-z0-9_ -]{2,60})\s*:/gm,
/<\/?([A-Za-z][A-Za-z0-9_-]{2,})\b[^>]*>/g,
/https?:\/\/[^\s"'(),}\]]+/gi,
/\b[A-Z]{2,}(?:-[A-Z0-9]+)*\b/g,
/\b[A-Za-z_][A-Za-z0-9_]*_[A-Za-z0-9_]+\b/g,
/\b[A-Za-z][A-Za-z0-9]+-[A-Za-z0-9-]+\b/g,
/\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/gi,
/\b\d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})?)?\b/g,
/\b\d+(?:[.,:/-]\d+)*(?:%|[A-Za-z]+)?\b/g,
/(?:^|\s)((?:\.{0,2}\/|\/)[\w@./-]{3,})/g
];
for (const pattern of patterns) {
for (const match of source.matchAll(pattern)) {
if (push(match[1] ?? match[0])) return anchors;
}
}
return anchors;
};
/**
* 将结构锚点渲染成 prompt 中的显式候选列表。
*
* 这不是普通关键词召回,只提醒模型优先原样保留字段名、路径、ID、数字等跨业务稳定信息。
*/
const renderExactAnchors = (content: string, limit?: number) => {
const anchors = extractExactAnchors(content, limit);
if (anchors.length === 0) return '';
return `<structural_anchor_candidates>
${anchors.map((anchor) => `- ${anchor}`).join('\n')}
</structural_anchor_candidates>
`;
};
/**
* 将历史消息格式化成 checkpoint 压缩模型可读的 JSON。
*
* 这里只暴露压缩需要的协议字段,避免把无关运行时字段塞进 prompt 增加 token 和噪声。
*/
const formatMessagesForCheckpoint = (messages: ChatCompletionMessageParam[]) => const formatMessagesForCheckpoint = (messages: ChatCompletionMessageParam[]) =>
JSON.stringify( JSON.stringify(
messages.map((message) => ({ messages.map((message) => ({
...@@ -14,13 +86,21 @@ const formatMessagesForCheckpoint = (messages: ChatCompletionMessageParam[]) => ...@@ -14,13 +86,21 @@ const formatMessagesForCheckpoint = (messages: ChatCompletionMessageParam[]) =>
); );
export const getCompressRequestMessagesPrompt = async () => { export const getCompressRequestMessagesPrompt = async () => {
return `你是 Agent 历史上下文 checkpoint 压缩专家。你的任务是把用户提供的对话历史压缩成一段可继续工作的上下文摘要 string。 return `你是 Agent 历史上下文 checkpoint 压缩专家。你的任务是把用户提供的对话历史压缩成一段可继续工作的高保真上下文摘要 string。
你的优先级顺序:
1. 保留后续继续任务必须依赖的信息。
2. 精确保留字段名、工具名、函数名、参数名、ID、路径、URL、错误码、数字、日期和明确约束。
3. 保留原文中的关键实体、对象、地点、人物、组织、状态、结果、原因、时间线和数值关系。
4. 删除寒暄、重复确认、无结论推理和低价值展开。
## 输出要求 ## 输出要求
1. 只输出一个 <context_checkpoint>...</context_checkpoint> 文本块。 1. 只输出一个 <context_checkpoint>...</context_checkpoint> 文本块。
2. 不要输出 JSON,不要输出 Markdown 代码块,不要输出解释。 2. 不要输出 JSON,不要输出 Markdown 代码块,不要输出解释。
3. 旧工具调用只总结关键结果,不保留 tool_call_id,不伪造工具执行。 3. 旧工具调用只总结关键输入、关键结果和失败原因;不要保留 tool_call_id,不伪造工具执行。
4. 所有小节都可以为空,但不要删除小节标题,便于后续机器读取。
5. 原文主要是中文就用中文输出,原文主要是英文就用英文输出;不要翻译名称、字段、ID、路径、URL、错误码、代码和工具参数。
## 必须保留的信息 ## 必须保留的信息
...@@ -28,9 +108,22 @@ export const getCompressRequestMessagesPrompt = async () => { ...@@ -28,9 +108,22 @@ export const getCompressRequestMessagesPrompt = async () => {
- 用户明确约束、偏好、禁止事项 - 用户明确约束、偏好、禁止事项
- 已完成的工作、已做决策、失败但影响后续选择的尝试 - 已完成的工作、已做决策、失败但影响后续选择的尝试
- 关键事实、数据、文件名、资源名、接口名、错误信息 - 关键事实、数据、文件名、资源名、接口名、错误信息
- 值得记住的工具结果和结论 - 工具调用得到的关键结果、失败原因和后续依赖
- 未解决问题、下一步应做事项 - 未解决问题、下一步应做事项
## 事实保留规则
- 用紧凑 bullet 保留“实体/对象 -> 属性/状态/结论/数值/原因”的事实关系,不要只写泛化结论。
- 人名、地名、组织名、产品名、资源名、字段值、标题、编号、金额、比例、时间、地址和错误文本要尽量原样保留。
- 对中文内容,不要把具体事实改写成“相关信息、若干项目、一些结果、该问题”等空泛表达。
- 如果多个事实同属一个主题,可以合并成一行,但不能丢掉具体实体和值。
## 结构锚点规则
- 对 structural_anchor_candidates 和 tool_call_memory 里的字段名、函数名、参数名、ID、路径、URL、错误码、数字、日期,优先原样保留。
- 不要为了保留锚点而堆无关关键词;锚点必须服务于后续执行、定位资源、复现工具结果或理解约束。
- 不要把具体工具名、机构名、文件名、接口名压成“相关工具/某机构/该文件/接口”。
## 忠实性要求 ## 忠实性要求
- 不要添加原文不存在的事实。 - 不要添加原文不存在的事实。
...@@ -52,6 +145,14 @@ export const getCompressRequestMessagesPrompt = async () => { ...@@ -52,6 +145,14 @@ export const getCompressRequestMessagesPrompt = async () => {
## Facts / Data To Preserve ## Facts / Data To Preserve
## Entity / Value / Relation Ledger
## Exact Keys / Labels / Names To Preserve
- tools/functions:
- argument_or_field_keys:
- ids/paths/urls:
- error_codes_or_numbers:
## Tool Results Worth Remembering ## Tool Results Worth Remembering
## Files / Resources Mentioned ## Files / Resources Mentioned
...@@ -63,37 +164,71 @@ export const getCompressRequestMessagesPrompt = async () => { ...@@ -63,37 +164,71 @@ export const getCompressRequestMessagesPrompt = async () => {
}; };
export const getCompressRequestMessagesUserPrompt = async ({ export const getCompressRequestMessagesUserPrompt = async ({
messages messages,
outputTokenLimit,
toolCallMemory
}: { }: {
messages: ChatCompletionMessageParam[]; messages: ChatCompletionMessageParam[];
outputTokenLimit?: number;
toolCallMemory?: string;
}) => { }) => {
const histories = formatMessagesForCheckpoint(messages);
return `<histories> return `<histories>
${formatMessagesForCheckpoint(messages)} ${histories}
</histories> </histories>
${outputTokenLimit ? `<output_budget>\nTarget maximum output tokens: ${outputTokenLimit}. Use compact bullets and omit nonessential prose.\n</output_budget>\n\n` : ''}${toolCallMemory ? `${toolCallMemory}\n\n` : ''}${renderExactAnchors(histories, 100)}
请执行历史上下文 checkpoint 压缩,只输出 <context_checkpoint>...</context_checkpoint>。 请执行历史上下文 checkpoint 压缩,只输出 <context_checkpoint>...</context_checkpoint>。
`; `;
}; };
export const getCompressLargeContentPrompt = async () => { export const getCompressLargeContentPrompt = async () => {
return `你是一个文本压缩专家。请在保留关键信息的前提下,尽量精简用户提供的文本。 return `你是一个生产场景通用文本压缩专家。请在保留后续使用所需事实的前提下,压缩用户提供的长文本。
核心目标:
- 保真优先于文风;只删除低价值信息,不添加原文不存在的信息。
- 优先保留稳定结构信息:标题、字段名、编号、ID、路径、URL、错误码、数字、日期、工具名、函数名、参数 key、返回字段 key。
- 同时保留语义事实信息:实体、对象、地点、人物、组织、状态、结果、原因、条件、时间线、数值和比较关系。
## 压缩原则 ## 压缩原则
1. 只能删除信息,不能添加原文不存在的信息。 1. 只能删除、概括和重组原文已有信息,不能新增事实。
2. 保留关键内容:数据、数字、名称、日期、核心结论、错误信息。 2. 保留核心结论、约束、状态、因果关系、失败原因、下一步依赖和支撑结论的关键事实。
3. 删除冗余:重复描述、冗长修饰语、空泛过渡句。 3. 用高密度 bullet 或短段落输出“主题 -> 事实/数值/关系”,不要只输出抽象摘要。
4. 精简表达:用简练语言、列表、概括替代详细说明。 4. 对 structural_anchor_candidates 中确实重要的结构锚点,优先原样保留。
5. 删除重复描述、修辞、铺垫、泛泛背景、例行说明和无新增事实的长句。
6. 原文主要是中文时使用中文,原文主要是英文时使用英文;不要翻译字段名、函数名、参数名、路径、URL、ID、错误码和专有名称。
7. 不要把具体人名、地名、组织名、产品名、资源名、标题、地址、金额、比例、日期、编号改写成“某人、某地、某机构、若干项目、相关数据”。
8. 如果原文是问答、报告、会议记录或多文档材料,优先保留可回答问题的事实,而不是只保留背景和结论。
## 输出要求 ## 输出要求
只输出压缩后的文本内容,不要包含解释、前后缀说明或 Markdown 代码块标记。`; 只输出压缩后的文本内容,不要包含解释、前后缀说明或 Markdown 代码块标记。
## 推荐输出形态
- 使用紧凑 bullet,每行保留一个主题或原文标签下的关键事实。
- 优先用原文标题、字段、问题、条目名称作为 bullet 前缀,再接具体事实、数值或结论。
- 预算紧张时先缩短解释性文字,再删除重复事实;不要先删除名称、标签、数字和结论。`;
}; };
export const getCompressLargeContentUserPrompt = async ({ content }: { content: string }) => { export const getCompressLargeContentUserPrompt = async ({
content,
outputTokenLimit
}: {
content: string;
outputTokenLimit?: number;
}) => {
return `<content> return `<content>
${content} ${content}
</content> </content>
${
outputTokenLimit
? `<output_budget>\nTarget maximum output tokens: ${outputTokenLimit}.
Use compact bullets; preserve original labels, names, numbers and conclusions first.\n</output_budget>\n\n`
: ''
}${renderExactAnchors(content)}
请执行压缩操作。`; 请执行压缩操作。`;
}; };
...@@ -41,6 +41,7 @@ import { ...@@ -41,6 +41,7 @@ import {
compressRequestMessages, compressRequestMessages,
compressToolResponse compressToolResponse
} from '@fastgpt/service/core/ai/llm/compress'; } from '@fastgpt/service/core/ai/llm/compress';
import { extractExactAnchors } from '@fastgpt/service/core/ai/llm/compress/prompt';
const model: LLMModelItemType = { const model: LLMModelItemType = {
type: ModelTypeEnum.llm, type: ModelTypeEnum.llm,
...@@ -93,6 +94,23 @@ const mockDefaultUsagePoints = () => { ...@@ -93,6 +94,23 @@ const mockDefaultUsagePoints = () => {
}); });
}; };
const mockPromptTokensForLlmCompression = ({
cleanedTokens = 1000,
finalTokens = 50,
initialTokens = 1000
}: {
cleanedTokens?: number;
finalTokens?: number;
initialTokens?: number;
} = {}) => {
countPromptTokensMock
.mockResolvedValueOnce(initialTokens)
.mockResolvedValueOnce(cleanedTokens)
.mockResolvedValueOnce(cleanedTokens)
.mockResolvedValueOnce(finalTokens)
.mockResolvedValue(initialTokens);
};
describe('compressRequestMessages', () => { describe('compressRequestMessages', () => {
beforeEach(() => { beforeEach(() => {
vi.clearAllMocks(); vi.clearAllMocks();
...@@ -154,7 +172,9 @@ describe('compressRequestMessages', () => { ...@@ -154,7 +172,9 @@ describe('compressRequestMessages', () => {
expect(compressPrompt).not.toContain('recent user 3'); expect(compressPrompt).not.toContain('recent user 3');
expect(userPrompt).toContain('<histories>'); expect(userPrompt).toContain('<histories>');
expect(userPrompt).toContain('recent user 3'); expect(userPrompt).toContain('recent user 3');
expect(userPrompt).toContain('<output_budget>');
expect(compressPrompt).not.toContain('最近消息预览'); expect(compressPrompt).not.toContain('最近消息预览');
expect(createLLMResponseMock.mock.calls[0][0].body.max_tokens).toBeUndefined();
}); });
it('should pass reasoning effort to the checkpoint compression LLM request', async () => { it('should pass reasoning effort to the checkpoint compression LLM request', async () => {
...@@ -224,6 +244,109 @@ describe('compressRequestMessages', () => { ...@@ -224,6 +244,109 @@ describe('compressRequestMessages', () => {
); );
}); });
it('should not replace an over-target LLM checkpoint when final messages stay within the production threshold', async () => {
countGptMessagesTokensMock.mockResolvedValueOnce(5000).mockResolvedValueOnce(2800);
createLLMResponseMock.mockResolvedValue({
answerText:
'<context_checkpoint>\n# Context Checkpoint\n## User Goal\n保留完整语义摘要,而不是首尾截断。\n</context_checkpoint>',
usage: {
inputTokens: 500,
outputTokens: 3000
},
requestId: 'req_soft_budget',
finish_reason: 'stop'
});
const result = await compressRequestMessages({
messages: createMessages(),
model
});
expect(result.contextCheckpoint).toContain('保留完整语义摘要');
expect(result.contextCheckpoint).not.toContain('## Source History Excerpts');
expect(createLLMResponseMock.mock.calls[0][0].body.max_tokens).toBeUndefined();
});
it('should use deterministic fallback only when the final compressed messages fit the threshold', async () => {
countGptMessagesTokensMock
.mockResolvedValueOnce(5000)
.mockResolvedValueOnce(4500)
.mockResolvedValueOnce(1200)
.mockResolvedValueOnce(3000)
.mockResolvedValueOnce(2600);
createLLMResponseMock.mockResolvedValue({
answerText: '<context_checkpoint>\n超长 LLM checkpoint\n</context_checkpoint>',
usage: {
inputTokens: 500,
outputTokens: 3000
},
requestId: 'req_deterministic_budget',
finish_reason: 'stop'
});
const messages = createMessages();
const result = await compressRequestMessages({
messages,
model
});
expect(result.contextCheckpoint).toContain('## Source History Excerpts');
expect(result.messages).not.toBe(messages);
expect(countGptMessagesTokensMock).toHaveBeenNthCalledWith(3, {
messages: [messages[0]]
});
});
it('should return original messages when compressed checkpoint still exceeds the production threshold', async () => {
countGptMessagesTokensMock
.mockResolvedValueOnce(5000)
.mockResolvedValueOnce(4500)
.mockResolvedValueOnce(4500)
.mockResolvedValueOnce(1200);
createLLMResponseMock.mockResolvedValue({
answerText: '<context_checkpoint>\n超长 tool checkpoint\n</context_checkpoint>',
usage: {
inputTokens: 500,
outputTokens: 3000
},
requestId: 'req_over_budget_tool_history',
finish_reason: 'stop'
});
const messages: ChatCompletionMessageParam[] = [
{
role: ChatCompletionRequestMessageRoleEnum.System,
content: 'system prompt'
},
{
role: ChatCompletionRequestMessageRoleEnum.User,
content: 'Run search_orders.'
},
{
role: ChatCompletionRequestMessageRoleEnum.Assistant,
content: null,
tool_calls: [
{
id: 'call_search_orders',
type: 'function',
function: {
name: 'search_orders',
arguments: '{"customerId":"c_123"}'
}
}
]
}
];
const result = await compressRequestMessages({
messages,
model
});
expect(result.messages).toBe(messages);
expect(result.contextCheckpoint).toBeUndefined();
expect(result.requestIds).toEqual(['req_over_budget_tool_history']);
});
it('should keep original messages when below compression threshold', async () => { it('should keep original messages when below compression threshold', async () => {
countGptMessagesTokensMock.mockResolvedValue(100); countGptMessagesTokensMock.mockResolvedValue(100);
...@@ -237,6 +360,172 @@ describe('compressRequestMessages', () => { ...@@ -237,6 +360,172 @@ describe('compressRequestMessages', () => {
expect(createLLMResponseMock).not.toHaveBeenCalled(); expect(createLLMResponseMock).not.toHaveBeenCalled();
}); });
it('should build a local structured checkpoint for over-threshold tool-call histories', async () => {
countGptMessagesTokensMock.mockResolvedValueOnce(5000).mockResolvedValueOnce(1200);
const messages: ChatCompletionMessageParam[] = [
{
role: ChatCompletionRequestMessageRoleEnum.System,
content: 'system prompt'
},
{
role: ChatCompletionRequestMessageRoleEnum.User,
content:
'Case alpha. Available tools: [{"type":"function","function":{"name":"search_orders","parameters":{"properties":{"customerId":{"type":"string"}}}}}]'
},
{
role: ChatCompletionRequestMessageRoleEnum.User,
content: 'Find recent orders for customer c_123.'
},
{
role: ChatCompletionRequestMessageRoleEnum.Assistant,
content: null,
tool_calls: [
{
id: 'call_search_orders',
type: 'function',
function: {
name: 'search_orders',
arguments: '{"customerId":"c_123","limit":5}'
}
}
]
}
];
const result = await compressRequestMessages({
messages,
model
});
expect(createLLMResponseMock).not.toHaveBeenCalled();
expect(result.usage).toBeUndefined();
expect(result.messages).toEqual([
messages[0],
{
role: ChatCompletionRequestMessageRoleEnum.User,
content: result.contextCheckpoint,
hideInUI: true
}
]);
expect(result.contextCheckpoint).toContain('<context_checkpoint>');
expect(result.contextCheckpoint).toContain('Case alpha. Available tools:');
expect(result.contextCheckpoint).toContain('search_orders');
expect(result.contextCheckpoint).toContain('"customerId":"c_123"');
expect(result.contextCheckpoint).toContain('Find recent orders for customer c_123');
});
it('should count system messages before accepting a local structured checkpoint', async () => {
countGptMessagesTokensMock
.mockResolvedValueOnce(5000)
.mockResolvedValueOnce(4800)
.mockResolvedValueOnce(260);
createLLMResponseMock.mockResolvedValue({
answerText: '<context_checkpoint>\nllm fallback summary\n</context_checkpoint>',
usage: {
inputTokens: 50,
outputTokens: 10
},
requestId: 'req_structured_with_system_budget',
finish_reason: 'stop'
});
const messages: ChatCompletionMessageParam[] = [
{
role: ChatCompletionRequestMessageRoleEnum.System,
content: 'very long system prompt'
},
{
role: ChatCompletionRequestMessageRoleEnum.User,
content:
'Available tools: [{"type":"function","function":{"name":"search_orders","parameters":{"properties":{"customerId":{"type":"string"}}}}}]'
},
{
role: ChatCompletionRequestMessageRoleEnum.Assistant,
content: null,
tool_calls: [
{
id: 'call_search_orders',
type: 'function',
function: {
name: 'search_orders',
arguments: '{"customerId":"c_123"}'
}
}
]
}
];
const result = await compressRequestMessages({
messages,
model
});
expect(createLLMResponseMock).toHaveBeenCalled();
expect(result.contextCheckpoint).toContain('llm fallback summary');
});
it('should include generic tool call memory in checkpoint compression prompt', async () => {
countGptMessagesTokensMock
.mockResolvedValueOnce(5000)
.mockResolvedValueOnce(5000)
.mockResolvedValueOnce(2000);
createLLMResponseMock.mockResolvedValue({
answerText: '<context_checkpoint>\ntool summary\n</context_checkpoint>',
usage: {
inputTokens: 50,
outputTokens: 10
},
requestId: 'req_tool_memory',
finish_reason: 'stop'
});
const messages: ChatCompletionMessageParam[] = [
{
role: ChatCompletionRequestMessageRoleEnum.System,
content: 'system prompt'
},
{
role: ChatCompletionRequestMessageRoleEnum.User,
content: 'Search enterprise contracts signed by Acme in 2025.'
},
{
role: ChatCompletionRequestMessageRoleEnum.Assistant,
content: null,
tool_calls: [
{
id: 'call_search_contracts',
type: 'function',
function: {
name: 'search_contracts',
arguments: '{"company":"Acme","year":2025}'
}
}
]
},
{
role: ChatCompletionRequestMessageRoleEnum.Tool,
tool_call_id: 'call_search_contracts',
content: '{"contracts":[{"id":"ctr_2025_001","amount":1200000}]}'
}
];
const result = await compressRequestMessages({
messages,
model
});
expect(createLLMResponseMock).toHaveBeenCalledTimes(1);
expect(result.messages).toHaveLength(2);
expect(result.messages[0]).toBe(messages[0]);
expect(result.contextCheckpoint).toBe(
'<context_checkpoint>\ntool summary\n</context_checkpoint>'
);
const userPrompt = createLLMResponseMock.mock.calls[0][0].body.messages[1].content;
expect(userPrompt).toContain('<tool_call_memory>');
expect(userPrompt).toContain('fn=search_contracts');
expect(userPrompt).toContain('args={"company":"Acme","year":2025}');
expect(userPrompt).toContain('user=Search enterprise contracts');
expect(userPrompt).toContain('"id":"ctr_2025_001"');
});
it('should use the full request context to decide checkpoint compression', async () => { it('should use the full request context to decide checkpoint compression', async () => {
createLLMResponseMock.mockResolvedValue({ createLLMResponseMock.mockResolvedValue({
answerText: '<context_checkpoint>\nshort user summary\n</context_checkpoint>', answerText: '<context_checkpoint>\nshort user summary\n</context_checkpoint>',
...@@ -457,7 +746,7 @@ describe('compressLargeContent', () => { ...@@ -457,7 +746,7 @@ describe('compressLargeContent', () => {
}); });
it('should use LLM chunk compression when rule cleanup is not enough', async () => { it('should use LLM chunk compression when rule cleanup is not enough', async () => {
countPromptTokensMock.mockResolvedValue(1000); mockPromptTokensForLlmCompression();
countGptMessagesTokensMock.mockResolvedValue(50); countGptMessagesTokensMock.mockResolvedValue(50);
createLLMResponseMock.mockResolvedValue({ createLLMResponseMock.mockResolvedValue({
answerText: ' compressed chunk ', answerText: ' compressed chunk ',
...@@ -475,8 +764,7 @@ describe('compressLargeContent', () => { ...@@ -475,8 +764,7 @@ describe('compressLargeContent', () => {
compressedTokenLimit: 100 compressedTokenLimit: 100
}); });
expect(result).toEqual({ expect(result).toMatchObject({
compressed: 'compressed chunk',
usage: { usage: {
moduleName: 'account_usage:llm_compress_text', moduleName: 'account_usage:llm_compress_text',
model: 'GPT-4', model: 'GPT-4',
...@@ -486,6 +774,7 @@ describe('compressLargeContent', () => { ...@@ -486,6 +774,7 @@ describe('compressLargeContent', () => {
}, },
requestIds: ['req_chunk'] requestIds: ['req_chunk']
}); });
expect(result.compressed).toBe('compressed chunk');
expect(createLLMResponseMock).toHaveBeenCalledWith( expect(createLLMResponseMock).toHaveBeenCalledWith(
expect.objectContaining({ expect.objectContaining({
body: expect.objectContaining({ body: expect.objectContaining({
...@@ -503,10 +792,13 @@ describe('compressLargeContent', () => { ...@@ -503,10 +792,13 @@ describe('compressLargeContent', () => {
expect(userPrompt).toContain('<content>'); expect(userPrompt).toContain('<content>');
expect(userPrompt).toContain('large content that requires LLM compression'); expect(userPrompt).toContain('large content that requires LLM compression');
expect(userPrompt).toContain('</content>'); expect(userPrompt).toContain('</content>');
expect(userPrompt).toContain('<output_budget>');
expect(userPrompt).toContain('Target maximum output tokens: 65');
expect(createLLMResponseMock.mock.calls[0][0].body.max_tokens).toBeUndefined();
}); });
it('should pass reasoning effort to large content compression requests', async () => { it('should pass reasoning effort to large content compression requests', async () => {
countPromptTokensMock.mockResolvedValue(1000); mockPromptTokensForLlmCompression();
countGptMessagesTokensMock.mockResolvedValue(50); countGptMessagesTokensMock.mockResolvedValue(50);
createLLMResponseMock.mockResolvedValue({ createLLMResponseMock.mockResolvedValue({
answerText: 'compressed', answerText: 'compressed',
...@@ -529,7 +821,7 @@ describe('compressLargeContent', () => { ...@@ -529,7 +821,7 @@ describe('compressLargeContent', () => {
}); });
it('should keep original chunk text when LLM returns empty chunk content', async () => { it('should keep original chunk text when LLM returns empty chunk content', async () => {
countPromptTokensMock.mockResolvedValue(1000); mockPromptTokensForLlmCompression();
countGptMessagesTokensMock.mockResolvedValue(50); countGptMessagesTokensMock.mockResolvedValue(50);
createLLMResponseMock.mockResolvedValue({ createLLMResponseMock.mockResolvedValue({
answerText: '', answerText: '',
...@@ -552,10 +844,17 @@ describe('compressLargeContent', () => { ...@@ -552,10 +844,17 @@ describe('compressLargeContent', () => {
}); });
it('should truncate merged LLM output when it still exceeds the compressed token limit', async () => { it('should truncate merged LLM output when it still exceeds the compressed token limit', async () => {
countPromptTokensMock.mockResolvedValue(1000); countPromptTokensMock
countGptMessagesTokensMock.mockResolvedValueOnce(999).mockResolvedValueOnce(1000); .mockResolvedValueOnce(1000)
.mockResolvedValueOnce(1000)
.mockResolvedValueOnce(1000)
.mockResolvedValueOnce(999)
.mockResolvedValueOnce(1000)
.mockResolvedValueOnce(999)
.mockResolvedValueOnce(999)
.mockResolvedValueOnce(50);
createLLMResponseMock.mockResolvedValue({ createLLMResponseMock.mockResolvedValue({
answerText: 'x'.repeat(100), answerText: 'x'.repeat(1000),
usage: { usage: {
inputTokens: 20, inputTokens: 20,
outputTokens: 200 outputTokens: 200
...@@ -569,7 +868,172 @@ describe('compressLargeContent', () => { ...@@ -569,7 +868,172 @@ describe('compressLargeContent', () => {
compressedTokenLimit: 100 compressedTokenLimit: 100
}); });
expect(result.compressed).toContain('... [content truncated] ...'); expect(result.compressed.length).toBeLessThan(1000);
expect(result.compressed.length).toBeGreaterThan(0);
});
it('should keep LLM merge output when tokens are within budget even if char length barely changes', async () => {
countPromptTokensMock
.mockResolvedValueOnce(1000)
.mockResolvedValueOnce(1000)
.mockResolvedValueOnce(1000)
.mockResolvedValueOnce(999)
.mockResolvedValueOnce(1000)
.mockResolvedValueOnce(80)
.mockResolvedValueOnce(80)
.mockResolvedValueOnce(80);
createLLMResponseMock
.mockResolvedValueOnce({
answerText: 'x'.repeat(1000),
usage: {
inputTokens: 20,
outputTokens: 200
},
requestId: 'req_initial_long'
})
.mockResolvedValueOnce({
answerText: 'y'.repeat(980),
usage: {
inputTokens: 20,
outputTokens: 80
},
requestId: 'req_merge_within_budget'
});
const result = await compressLargeContent({
content: 'large content',
model,
compressedTokenLimit: 100
});
expect(result.compressed).toBe('y'.repeat(980));
expect(result.compressed).not.toContain('content truncated');
});
it('should append source excerpts when LLM output uses too little of the budget', async () => {
countPromptTokensMock
.mockResolvedValueOnce(2000)
.mockResolvedValueOnce(2000)
.mockResolvedValueOnce(2000)
.mockResolvedValueOnce(100)
.mockResolvedValueOnce(2000)
.mockResolvedValueOnce(100)
.mockResolvedValueOnce(700)
.mockResolvedValue(700);
createLLMResponseMock.mockResolvedValue({
answerText: '简短摘要',
usage: {
inputTokens: 20,
outputTokens: 20
},
requestId: 'req_short_summary'
});
const result = await compressLargeContent({
content: ['开头字段:关键背景', '正文内容。'.repeat(400), '尾部字段:最终结论'].join('\n'),
model,
compressedTokenLimit: 1000
});
expect(result.compressed).toContain('简短摘要');
expect(result.compressed).toContain('Source excerpts for exact labels and facts');
expect(result.compressed).toContain('尾部字段');
});
it('should append exact source anchors while staying within the token budget', async () => {
countPromptTokensMock
.mockResolvedValueOnce(2000)
.mockResolvedValueOnce(2000)
.mockResolvedValueOnce(2000)
.mockResolvedValueOnce(500)
.mockResolvedValueOnce(2000)
.mockResolvedValueOnce(500)
.mockResolvedValueOnce(540)
.mockResolvedValueOnce(580)
.mockResolvedValueOnce(600);
createLLMResponseMock.mockResolvedValue({
answerText: '压缩后的核心事实',
usage: {
inputTokens: 20,
outputTokens: 20
},
requestId: 'req_anchor_append'
});
const result = await compressLargeContent({
content: ['问题标题:关键问题', '字段名称:重要字段', '正文内容。'.repeat(400)].join('\n'),
model,
compressedTokenLimit: 1000
});
expect(result.compressed).toContain('压缩后的核心事实');
expect(result.compressed).toContain('Source labels / exact anchors');
expect(result.compressed).toContain('问题标题');
expect(result.compressed).toContain('字段名称');
});
it('should skip source anchors when compressed output already uses most of the budget', async () => {
countPromptTokensMock
.mockResolvedValueOnce(2000)
.mockResolvedValueOnce(2000)
.mockResolvedValueOnce(2000)
.mockResolvedValueOnce(810)
.mockResolvedValueOnce(2000)
.mockResolvedValueOnce(810)
.mockResolvedValueOnce(810);
createLLMResponseMock.mockResolvedValue({
answerText: '压缩后的核心事实',
usage: {
inputTokens: 20,
outputTokens: 20
},
requestId: 'req_anchor_skip'
});
const result = await compressLargeContent({
content: ['问题标题:关键问题', '字段名称:重要字段', '正文内容。'.repeat(400)].join('\n'),
model,
compressedTokenLimit: 1000
});
expect(result.compressed).toBe('压缩后的核心事实');
expect(result.compressed).not.toContain('Source labels / exact anchors');
});
it('should append at most twelve source anchors', async () => {
countPromptTokensMock
.mockResolvedValueOnce(2000)
.mockResolvedValueOnce(2000)
.mockResolvedValueOnce(2000)
.mockResolvedValueOnce(500)
.mockResolvedValueOnce(2000)
.mockResolvedValueOnce(500)
.mockResolvedValue(520);
createLLMResponseMock.mockResolvedValue({
answerText: '压缩后的核心事实',
usage: {
inputTokens: 20,
outputTokens: 20
},
requestId: 'req_anchor_cap'
});
const result = await compressLargeContent({
content: [
...Array.from({ length: 20 }, (_, index) => `字段${index + 1}:值${index + 1}`),
'正文内容。'.repeat(400)
].join('\n'),
model,
compressedTokenLimit: 1000
});
const appendedAnchorCount =
result.compressed
.split('Source labels / exact anchors:')[1]
?.split('\n')
.filter((line) => line.trim().startsWith('- ')).length ?? 0;
expect(appendedAnchorCount).toBeLessThanOrEqual(12);
}); });
it('should return cleaned content when LLM chunk compression throws', async () => { it('should return cleaned content when LLM chunk compression throws', async () => {
...@@ -588,7 +1052,7 @@ describe('compressLargeContent', () => { ...@@ -588,7 +1052,7 @@ describe('compressLargeContent', () => {
}); });
it('should skip billing points for chunk compression when valid userKey is provided', async () => { it('should skip billing points for chunk compression when valid userKey is provided', async () => {
countPromptTokensMock.mockResolvedValue(1000); mockPromptTokensForLlmCompression();
countGptMessagesTokensMock.mockResolvedValue(50); countGptMessagesTokensMock.mockResolvedValue(50);
createLLMResponseMock.mockResolvedValue({ createLLMResponseMock.mockResolvedValue({
answerText: 'compressed', answerText: 'compressed',
...@@ -615,6 +1079,40 @@ describe('compressLargeContent', () => { ...@@ -615,6 +1079,40 @@ describe('compressLargeContent', () => {
}); });
}); });
describe('extractExactAnchors', () => {
it('should extract only generic structural anchors instead of ordinary keywords', () => {
const anchors = extractExactAnchors(
[
'The ordinary project background should not become an anchor.',
'tool_name: search_contracts',
'trace_id: req_2025_001',
'问题标题:如何处理长文本压缩',
'## Release Notes',
'1.2 处理流程',
'请参考【结论摘要】继续执行。',
'Use /tmp/project/report.txt on 2025-01-02.',
'statusCode: 429'
].join('\n'),
20
);
expect(anchors).toEqual(
expect.arrayContaining([
'tool_name',
'trace_id',
'问题标题',
'Release Notes',
'处理流程',
'结论摘要',
'req_2025_001',
'2025-01-02',
'429'
])
);
expect(anchors).not.toEqual(expect.arrayContaining(['ordinary', 'project', 'background']));
});
});
describe('compressToolResponse', () => { describe('compressToolResponse', () => {
beforeEach(() => { beforeEach(() => {
vi.clearAllMocks(); vi.clearAllMocks();
...@@ -633,8 +1131,143 @@ describe('compressToolResponse', () => { ...@@ -633,8 +1131,143 @@ describe('compressToolResponse', () => {
expect(createLLMResponseMock).not.toHaveBeenCalled(); expect(createLLMResponseMock).not.toHaveBeenCalled();
}); });
it('should minify JSON tool responses without LLM when minified content fits the budget', async () => {
countPromptTokensMock.mockResolvedValue(100);
const response = JSON.stringify(
{
source: 'tool_call_log',
rows: [
{
id: 'multiple_001',
messages: [
{
role: 'user',
content: 'Find lawsuits filed against Google in California in 2020.'
},
{
role: 'assistant',
content: null,
tool_calls: [
{
type: 'function',
function: {
name: 'lawsuits_search',
arguments: '{"company_name":"Google","location":"California","year":2020}'
}
}
]
}
],
tools: [
{
type: 'function',
function: {
name: 'lawsuits_search',
description: 'Long description should be removed from compressed tool schema.',
parameters: {
type: 'object',
required: ['company_name', 'location', 'year'],
properties: {
company_name: {
type: 'string',
description: 'Company name.'
},
location: {
type: 'string',
description: 'Location.'
},
year: {
type: 'integer',
description: 'Year.'
}
}
}
}
}
]
}
]
},
null,
2
);
const result = await compressToolResponse({
response,
model,
compressedTokenLimit: 1200,
currentMessagesTokens: 0,
toolLength: 1
});
expect(createLLMResponseMock).not.toHaveBeenCalled();
const compressed = JSON.parse(result.compressed);
expect(compressed.source).toBe('tool_call_log');
expect(result.compressed).toContain('rows');
expect(result.compressed).toContain('lawsuits_search');
expect(result.compressed).toContain('company_name');
expect(result.compressed).toContain('California');
expect(result.compressed).toContain('Long description');
expect(result.compressed).not.toContain('\n');
});
it('should summarize larger JSON tool responses structurally without LLM', async () => {
countPromptTokensMock.mockResolvedValueOnce(900).mockResolvedValueOnce(180);
const response = JSON.stringify({
source: 'tool_call_log',
rows: [
{
id: 'multiple_001',
messages: [
{
role: 'user',
content: 'Find lawsuits filed against Google in California in 2020.'
}
],
tools: [
{
type: 'function',
function: {
name: 'lawsuits_search',
parameters: {
type: 'object',
required: ['company_name', 'location', 'year']
}
}
}
]
},
{
id: 'multiple_002',
messages: [],
tools: []
}
]
});
const result = await compressToolResponse({
response,
model,
compressedTokenLimit: 1200,
currentMessagesTokens: 0,
toolLength: 1
});
expect(createLLMResponseMock).not.toHaveBeenCalled();
expect(result.compressed).toContain('JSON structural summary');
expect(result.compressed).toContain('root keys: source, rows');
expect(result.compressed).toContain('important scalar values: tool_call_log; multiple_001');
expect(result.compressed).toContain('rows: array(length=2)');
expect(result.compressed).toContain('rows[0].id: multiple_001');
expect(result.compressed).toContain('rows[0].tools[0].function.name: lawsuits_search');
expect(result.compressed).not.toContain('{"source"');
});
it('should use dynamic available context as the tool compressed token limit', async () => { it('should use dynamic available context as the tool compressed token limit', async () => {
countPromptTokensMock.mockResolvedValue(1600); mockPromptTokensForLlmCompression({
cleanedTokens: 1600,
initialTokens: 1600
});
createLLMResponseMock.mockResolvedValue({ createLLMResponseMock.mockResolvedValue({
answerText: 'compressed tool response', answerText: 'compressed tool response',
usage: { usage: {
...@@ -658,4 +1291,31 @@ describe('compressToolResponse', () => { ...@@ -658,4 +1291,31 @@ describe('compressToolResponse', () => {
expect(result.requestIds).toEqual(['req_tool']); expect(result.requestIds).toEqual(['req_tool']);
expect(createLLMResponseMock.mock.calls[0][0].body.reasoning_effort).toBe('high'); expect(createLLMResponseMock.mock.calls[0][0].body.reasoning_effort).toBe('high');
}); });
it('should respect caller provided compressed token limit for tool response', async () => {
mockPromptTokensForLlmCompression({
cleanedTokens: 1200,
initialTokens: 1200
});
createLLMResponseMock.mockResolvedValue({
answerText: 'budgeted tool response',
usage: {
inputTokens: 30,
outputTokens: 6
},
requestId: 'req_tool_budget'
});
const result = await compressToolResponse({
response: 'tool response',
model,
compressedTokenLimit: 1000,
currentMessagesTokens: 0,
toolLength: 1
});
expect(createLLMResponseMock).toHaveBeenCalledTimes(1);
expect(result.compressed).toBe('budgeted tool response');
expect(createLLMResponseMock.mock.calls[0][0].body.max_tokens).toBeUndefined();
});
}); });
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment