482 lines
17 KiB
JavaScript
482 lines
17 KiB
JavaScript
export function formatMessages(messages, proxyModel, randomFileName) {
|
||
// 检查是否是 Claude 模型
|
||
const isClaudeModel = proxyModel.toLowerCase().includes('claude');
|
||
|
||
// 启用特殊前缀
|
||
const USE_BACKSPACE_PREFIX = process.env.USE_BACKSPACE_PREFIX === 'true';
|
||
|
||
// 定义角色映射
|
||
const roleFeatures = getRoleFeatures(messages, isClaudeModel, USE_BACKSPACE_PREFIX);
|
||
|
||
// 清除第一条消息的 role
|
||
messages = clearFirstMessageRole(messages);
|
||
|
||
messages = removeCustomRoleDefinitions(messages);
|
||
|
||
messages = convertRoles(messages, roleFeatures);
|
||
|
||
// 替换 content 中的角色
|
||
messages = replaceRolesInContent(messages, roleFeatures);
|
||
|
||
// 如果启用 clewd
|
||
const CLEWD_ENABLED = process.env.CLEWD_ENABLED === 'true';
|
||
if (CLEWD_ENABLED) {
|
||
messages = xmlPlotAllMessages(messages, roleFeatures);
|
||
}
|
||
|
||
const hasAIRound0 = messages.some(message => message.content.includes('<!-- AI Round 0 begins. -->'));
|
||
|
||
let formattedMessages = messages.slice();
|
||
|
||
// 如果没有找到 AI Round 0 标记，直接返回处理后的消息数组
|
||
if (!hasAIRound0) {
|
||
return formattedMessages;
|
||
}
|
||
|
||
let userRoundCounter = 0;
|
||
let assistantRoundCounter = 0;
|
||
let descriptionPointCounter = 0;
|
||
let isFirstUserFound = false;
|
||
let lastAssistantRound = 0;
|
||
|
||
// 查找初始回合数
|
||
let initialRound = 0;
|
||
for (let i = 0; i < formattedMessages.length; i++) {
|
||
if (formattedMessages[i].role === roleFeatures.userRole) {
|
||
const nextMessage = formattedMessages[i + 1];
|
||
if (nextMessage && nextMessage.role === roleFeatures.assistantRole) {
|
||
const match = nextMessage.content.match(/<!-- AI Round (\d+) begins\. -->/);
|
||
if (match) {
|
||
initialRound = parseInt(match[1]);
|
||
userRoundCounter = initialRound - 1;
|
||
assistantRoundCounter = initialRound;
|
||
lastAssistantRound = initialRound;
|
||
descriptionPointCounter = 1;
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// 找到最后一个有效的 user 消息索引
|
||
let lastUserIndex = -1;
|
||
let contextEndIndex = formattedMessages.length;
|
||
for (let i = formattedMessages.length - 1; i >= 0; i--) {
|
||
if (formattedMessages[i].content.includes('</context> ---')) {
|
||
contextEndIndex = i;
|
||
}
|
||
if (formattedMessages[i].role === roleFeatures.userRole && lastUserIndex === -1) {
|
||
lastUserIndex = i;
|
||
}
|
||
if (lastUserIndex !== -1 && contextEndIndex !== formattedMessages.length) {
|
||
break;
|
||
}
|
||
}
|
||
|
||
let processedMessages = [];
|
||
for (let i = 0; i < formattedMessages.length; i++) {
|
||
const message = formattedMessages[i];
|
||
|
||
if (message.content.includes('<!-- AI Round 0 begins. -->')) {
|
||
processedMessages.push({
|
||
role: message.role,
|
||
content: message.content.replace('<!-- AI Round 0 begins. -->', '<建立记忆区>\n<!-- AI Round 0 begins. -->')
|
||
});
|
||
continue;
|
||
}
|
||
|
||
if (message.role === roleFeatures.userRole && i <= lastUserIndex) {
|
||
if (isFirstUserFound) {
|
||
userRoundCounter = lastAssistantRound + 1;
|
||
descriptionPointCounter++;
|
||
} else {
|
||
isFirstUserFound = true;
|
||
}
|
||
|
||
let roundInfo = '';
|
||
if (i === lastUserIndex) {
|
||
roundInfo = `最新${roleFeatures.userRole}:(${userRoundCounter})回合|${roleFeatures.assistantRole}:(${userRoundCounter + 1})回合开始，参考 <Human_inputs>，关联所有记忆重构语境时空关系的碎片:\n`;
|
||
} else {
|
||
const nextAssistantRound = userRoundCounter + 1;
|
||
roundInfo = `{{ 第 ${roleFeatures.userRole} = 回合${userRoundCounter}|${roleFeatures.assistantRole} = 回合${nextAssistantRound} 开始建立记忆区: [${descriptionPointCounter}]\n`;
|
||
}
|
||
message.content = roundInfo + message.content;
|
||
} else if (message.role === roleFeatures.assistantRole && i < lastUserIndex) {
|
||
const match = message.content.match(/<!-- AI Round (\d+) begins\. -->/);
|
||
if (match) {
|
||
assistantRoundCounter = parseInt(match[1]);
|
||
lastAssistantRound = assistantRoundCounter;
|
||
}
|
||
|
||
if (message.content.includes('<CHAR_turn>')) {
|
||
message.content += `\n}}\n<-- 记忆区 [${descriptionPointCounter}] 结束 -->`;
|
||
}
|
||
}
|
||
|
||
processedMessages.push(message);
|
||
}
|
||
|
||
return processedMessages;
|
||
}
|
||
|
||
/**
|
||
* 将首条消息role置空
|
||
* @param {Array} messages - 消息数组
|
||
* @returns {Array} 处理后的消息数组
|
||
*/
|
||
function clearFirstMessageRole(messages) {
|
||
if (!Array.isArray(messages) || messages.length === 0) {
|
||
return messages;
|
||
}
|
||
const processedMessages = messages.map(msg => ({...msg}));
|
||
|
||
processedMessages[0] = {
|
||
...processedMessages[0],
|
||
role: ''
|
||
};
|
||
|
||
return processedMessages;
|
||
}
|
||
|
||
// 转义特殊字符
|
||
function escapeRegExp(string) {
|
||
return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||
}
|
||
|
||
// 获取角色特征
|
||
function getRoleFeatures(messages, isClaudeModel, useBackspacePrefix) {
|
||
let prefix = useBackspacePrefix ? '\u0008' : '';
|
||
let systemRole = `${prefix}${isClaudeModel ? 'System' : 'system'}`;
|
||
let userRole = `${prefix}${isClaudeModel ? 'Human' : 'user'}`;
|
||
let assistantRole = `${prefix}${isClaudeModel ? 'Assistant' : 'assistant'}`;
|
||
|
||
// 匹配自定义角色
|
||
const rolePattern = /\[\|(\w+)::(.*?)\|\]/g;
|
||
let customRoles = {};
|
||
messages.forEach(message => {
|
||
let content = message.content;
|
||
let match;
|
||
while ((match = rolePattern.exec(content)) !== null) {
|
||
const roleKey = match[1]; // 'system', 'user', 'assistant'
|
||
customRoles[roleKey.toLowerCase()] = match[2];
|
||
}
|
||
});
|
||
|
||
if (Object.keys(customRoles).length > 0) {
|
||
prefix = '';
|
||
|
||
if (customRoles['system']) {
|
||
systemRole = customRoles['system'];
|
||
}
|
||
|
||
if (customRoles['user']) {
|
||
userRole = customRoles['user'];
|
||
}
|
||
|
||
if (customRoles['assistant']) {
|
||
assistantRole = customRoles['assistant'];
|
||
}
|
||
}
|
||
|
||
return {
|
||
systemRole,
|
||
userRole,
|
||
assistantRole,
|
||
prefix
|
||
};
|
||
}
|
||
|
||
// 移除 messages 自定义角色格式
|
||
function removeCustomRoleDefinitions(messages) {
|
||
const rolePattern = /\[\|\w+::.*?\|]/g;
|
||
|
||
return messages.map(message => {
|
||
let newContent = message.content.replace(rolePattern, '');
|
||
return {
|
||
...message,
|
||
content: newContent
|
||
};
|
||
});
|
||
}
|
||
|
||
// 转换角色
|
||
function convertRoles(messages, roleFeatures) {
|
||
const {systemRole, userRole, assistantRole} = roleFeatures;
|
||
const roleMap = {
|
||
'system': systemRole,
|
||
'user': userRole,
|
||
'human': userRole,
|
||
'assistant': assistantRole
|
||
};
|
||
|
||
return messages.map(message => {
|
||
let currentRole = message.role;
|
||
|
||
if (currentRole.startsWith('\u0008')) {
|
||
// 包含前缀不需要转换
|
||
return message;
|
||
} else {
|
||
const roleKey = currentRole.toLowerCase();
|
||
const newRole = roleMap[roleKey] || currentRole;
|
||
return {...message, role: newRole};
|
||
}
|
||
});
|
||
}
|
||
|
||
// 替换 content 中的角色定义
|
||
function replaceRolesInContent(messages, roleFeatures) {
|
||
// 避免重复添加
|
||
const roleMap = {
|
||
'System:': roleFeatures.systemRole.replace(roleFeatures.prefix, '') + ':',
|
||
'system:': roleFeatures.systemRole.replace(roleFeatures.prefix, '') + ':',
|
||
'Human:': roleFeatures.userRole.replace(roleFeatures.prefix, '') + ':',
|
||
'human:': roleFeatures.userRole.replace(roleFeatures.prefix, '') + ':',
|
||
'user:': roleFeatures.userRole.replace(roleFeatures.prefix, '') + ':',
|
||
'Assistant:': roleFeatures.assistantRole.replace(roleFeatures.prefix, '') + ':',
|
||
'assistant:': roleFeatures.assistantRole.replace(roleFeatures.prefix, '') + ':',
|
||
};
|
||
|
||
// 构建角色正则
|
||
const escapedLabels = Object.keys(roleMap).map(label => escapeRegExp(label));
|
||
const prefixPattern = roleFeatures.prefix ? escapeRegExp(roleFeatures.prefix) : '';
|
||
|
||
const roleNamesPattern = new RegExp(`(\\n\\n)(${prefixPattern})?(${escapedLabels.join('|')})`, 'g');
|
||
|
||
return messages.map(message => {
|
||
let newContent = message.content;
|
||
|
||
if (typeof newContent === 'string') {
|
||
// 仅替换段落开头角色
|
||
newContent = newContent.replace(roleNamesPattern, (match, p1, p2, p3) => {
|
||
const newRoleLabel = roleMap[p3] || p3;
|
||
const prefixToUse = p2 !== undefined ? p2 : roleFeatures.prefix;
|
||
return p1 + (prefixToUse || '') + newRoleLabel;
|
||
});
|
||
} else {
|
||
console.warn('message.content is not a string:', newContent);
|
||
newContent = '';
|
||
}
|
||
|
||
return {
|
||
...message,
|
||
content: newContent
|
||
};
|
||
});
|
||
}
|
||
|
||
/**
|
||
* xmlPlotAllMessages(messages, roleFeatures, options)
|
||
*
|
||
* 在 CLEWD 阶段，对一组 messages 进行二次处理：
|
||
* 默认将所有消息的 role 设置为空字符串（可以视为“去掉角色”）。
|
||
* 若 message.content 中含有 <|KEEP_ROLE|> 标记，则保留原先的 role，不置空。
|
||
* 通过 options.skipSystem = true/false，决定是否跳过 system 段落处理。
|
||
* 调用 xmlPlot(...) 时传入 apiKey / skipSystem 等参数，以进一步根据老版逻辑进行区分。
|
||
*
|
||
* @param {Array} messages - [{ role: 'user'/..., content: '...' }, ...]
|
||
* @param {object} roleFeatures - { systemRole, userRole, assistantRole, prefix }
|
||
* @return {Array} 新的 messages（content 已经过多轮正则处理）
|
||
* -----------------------------------------------------------------------------------
|
||
*/
|
||
export function xmlPlotAllMessages(messages, roleFeatures) {
|
||
return messages.map(msg => {
|
||
if (!msg.content.includes('<|KEEP_ROLE|>')) {
|
||
msg = {
|
||
...msg,
|
||
role: '' // 置空
|
||
};
|
||
}
|
||
const newContent = xmlPlot(msg.content, roleFeatures);
|
||
|
||
return {
|
||
...msg,
|
||
content: newContent
|
||
};
|
||
});
|
||
}
|
||
|
||
/**
|
||
* 对单条文本 content 进行多轮正则及合并处理
|
||
* @param {string} content - 消息体文本
|
||
* @param {object} roleFeatures - { systemRole, userRole, assistantRole, prefix }
|
||
*/
|
||
function xmlPlot(content, roleFeatures) {
|
||
let regexLog = '';
|
||
// 第一次正则替换
|
||
content = xmlPlot_regex(content, 1, regexLog);
|
||
|
||
// 第一次角色合并
|
||
const mergeTag = {
|
||
all: !content.includes('<|Merge Disable|>'),
|
||
system: !content.includes('<|Merge System Disable|>'),
|
||
human: !content.includes('<|Merge Human Disable|>'),
|
||
assistant: !content.includes('<|Merge Assistant Disable|>')
|
||
};
|
||
content = xmlPlot_merge(content, mergeTag, roleFeatures);
|
||
|
||
// 处理内嵌 <@N> ... </@N> 插入
|
||
const escapeRegExp = (str) => str.replace(/[\b.*+?^${}()|[\]\\]/g, '\\$&');
|
||
const humanLabelRaw = `${roleFeatures.userRole}:`;
|
||
const assistantLabelRaw = `${roleFeatures.assistantRole}:`;
|
||
const humanLabel = escapeRegExp(humanLabelRaw);
|
||
const assistantLabel = escapeRegExp(assistantLabelRaw);
|
||
|
||
// 根据段落分隔符拆分
|
||
let splitContent = content.split(new RegExp(`\\n\\n(?=${humanLabel}|${assistantLabel})`, 'g'));
|
||
let match;
|
||
while ((match = /<@(\d+)>(.*?)<\/@\1>/gs.exec(content)) !== null) {
|
||
let insertionIndex = splitContent.length - parseInt(match[1], 10) - 1;
|
||
if (insertionIndex >= 0) {
|
||
splitContent[insertionIndex] += '\n\n' + match[2];
|
||
}
|
||
content = content.replace(match[0], '');
|
||
}
|
||
content = splitContent.join('\n\n').replace(/<@(\d+)>.*?<\/@\1>/gs, '');
|
||
|
||
// 第二次正则替换
|
||
content = xmlPlot_regex(content, 2, regexLog);
|
||
|
||
// 第二次角色合并
|
||
content = xmlPlot_merge(content, mergeTag, roleFeatures);
|
||
|
||
// Plain Prompt 处理
|
||
const humanLabelPattern = new RegExp(`\\n\\n${humanLabel}`, 'g');
|
||
let segcontentHuman = content.split(humanLabelPattern);
|
||
let segcontentlastIndex = segcontentHuman.length - 1;
|
||
if (
|
||
segcontentlastIndex >= 2 &&
|
||
segcontentHuman[segcontentlastIndex].includes('<|Plain Prompt Enable|>') &&
|
||
!content.includes(`\n\nPlainPrompt:`)
|
||
) {
|
||
content = segcontentHuman
|
||
.slice(0, segcontentlastIndex)
|
||
.join(`\n\n${humanLabelRaw}`) +
|
||
`\n\nPlainPrompt:` +
|
||
segcontentHuman
|
||
.slice(segcontentlastIndex)
|
||
.join(`\n\n${humanLabelRaw}`)
|
||
.replace(new RegExp(`\\n\\n${humanLabel}\\s*PlainPrompt:`, 'g'), '\n\nPlainPrompt:');
|
||
}
|
||
|
||
// 第三次正则替换
|
||
content = xmlPlot_regex(content, 3, regexLog);
|
||
|
||
// 清理和格式化
|
||
content = content
|
||
// 移除剩余 <regex ...> 包裹
|
||
.replace(/<regex( +order *= *\d)?>.*?<\/regex>/gm, '')
|
||
// 统一换行
|
||
.replace(/\r\n|\r/gm, '\n')
|
||
// <|curtail|> 替换为换行
|
||
.replace(/\s*<\|curtail\|>\s*/g, '\n')
|
||
// <|join|> 去掉
|
||
.replace(/\s*<\|join\|>\s*/g, '')
|
||
// <|space|> 替换为" "
|
||
.replace(/\s*<\|space\|>\s*/g, ' ')
|
||
// 修正多余的空格/换行
|
||
.replace(new RegExp(`\\s*\\n\\n(${humanLabel}|${assistantLabel})\\s+`, 'g'), '\n\n$1 ')
|
||
// 对 <|xxx|> 做 JSON.parse 反序列化
|
||
.replace(/<\|(\\.*?)\|>/g, function (m, p1) {
|
||
try {
|
||
return JSON.parse(`"${p1.replace(/\\?"/g, '\\"')}"`);
|
||
} catch {
|
||
return m;
|
||
}
|
||
})
|
||
// 最后去掉多余
|
||
.replace(/\s*<\|(?!padtxt).*?\|>\s*/g, '\n\n')
|
||
.trim()
|
||
.replace(/(?<=\n)\n(?=\n)/g, '');
|
||
|
||
return content;
|
||
}
|
||
|
||
/**
|
||
* 解析 <regex>"/pattern/flags":"replacement"</regex> 标签并执行替换
|
||
* @param {string} content
|
||
* @param {number} order
|
||
* @param {string} regexLog
|
||
* @returns {string} 替换后的文本
|
||
*/
|
||
function xmlPlot_regex(content, order, regexLog) {
|
||
// 只匹配与当前 order 相符的 <regex> 标签
|
||
const patternRegex = new RegExp(
|
||
`<regex(?: +order *= *(${order}))?>\\s*"\\/([^"]*?)\\/([gimsyu]*)"\\s*:\\s*"(.*?)"\\s*<\\/regex>`,
|
||
'gm'
|
||
);
|
||
|
||
let match;
|
||
while ((match = patternRegex.exec(content)) !== null) {
|
||
// match[0] : <regex>...</regex>
|
||
// match[1] : order
|
||
// match[2] : pattern
|
||
// match[3] : flags
|
||
// match[4] : replacement
|
||
|
||
const fullBlock = match[0];
|
||
const subPattern = match[2];
|
||
const subFlags = match[3];
|
||
let replacement = match[4];
|
||
|
||
regexLog += fullBlock + '\n';
|
||
|
||
try {
|
||
// 构造 JS 正则
|
||
const regObj = new RegExp(subPattern, subFlags);
|
||
// 反序列化 replacement
|
||
replacement = JSON.parse(`"${replacement.replace(/\\?"/g, '\\"')}"`);
|
||
// 执行替换
|
||
content = content.replace(regObj, replacement);
|
||
} catch (err) {
|
||
console.log(`Regex error: ` + fullBlock + '\n' + err);
|
||
}
|
||
}
|
||
return content;
|
||
}
|
||
|
||
/**
|
||
* 多段 "Human:..." 或 "Assistant:..." 合并
|
||
* @param {string} content
|
||
* @param {*} mergeTag { all, system, human, assistant }
|
||
* @param {*} roleFeatures
|
||
* @returns {string}
|
||
*/
|
||
function xmlPlot_merge(content, mergeTag, roleFeatures) {
|
||
const escapeRegExp = (str) => str.replace(/[\b.*+?^${}()|[\]\\]/g, '\\$&');
|
||
|
||
const humanLabelRaw = `${roleFeatures.userRole}:`;
|
||
const assistantLabelRaw = `${roleFeatures.assistantRole}:`;
|
||
const humanLabel = escapeRegExp(humanLabelRaw);
|
||
const assistantLabel = escapeRegExp(assistantLabelRaw);
|
||
|
||
// 如果出现 xmlPlot:
|
||
if (/(\n\n|^\s*)xmlPlot:\s*/.test(content)) {
|
||
content = content.replace(
|
||
/(\n\n|^\s*)xmlPlot:\s*/g,
|
||
mergeTag.system && mergeTag.human && mergeTag.all
|
||
? `\n\n${humanLabelRaw} `
|
||
: '$1'
|
||
);
|
||
}
|
||
|
||
// 合并 Human 段
|
||
if (mergeTag.all && mergeTag.human) {
|
||
const humanRegex = new RegExp(`(?:\\n\\n|^\\s*)${humanLabel}(.*?)(?=\\n\\n(?:${assistantLabel}|$))`, 'gs');
|
||
content = content.replace(humanRegex, (match, p1) => {
|
||
const innerHumanLabelRegex = new RegExp(`\\n\\n${humanLabel}\\s*`, 'g');
|
||
return `\n\n${humanLabelRaw}` + p1.replace(innerHumanLabelRegex, '\n\n');
|
||
});
|
||
}
|
||
|
||
// 合并 Assistant 段
|
||
if (mergeTag.all && mergeTag.assistant) {
|
||
const assistantRegex = new RegExp(`(?:\\n\\n|^\\s*)${assistantLabel}(.*?)(?=\\n\\n(?:${humanLabel}|$))`, 'gs');
|
||
content = content.replace(assistantRegex, (match, p1) => {
|
||
const innerAssistantLabelRegex = new RegExp(`\\n\\n${assistantLabel}\\s*`, 'g');
|
||
return `\n\n${assistantLabelRaw}` + p1.replace(innerAssistantLabelRegex, '\n\n');
|
||
});
|
||
}
|
||
|
||
return content;
|
||
}
|