From 4ec827a9d34735714d4a6c40edeff4efd903bcc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=A2=AB=E9=81=97=E5=BF=98=E7=9A=84=E8=AE=B0=E5=BF=86?= <32097237+YIWANG-sketch@users.noreply.github.com> Date: Tue, 18 Feb 2025 12:49:58 +0800 Subject: [PATCH] optimize polling strategy --- you_providers/cookieUpdater.mjs | 587 ++++++++++++++++++++++---------- 1 file changed, 403 insertions(+), 184 deletions(-) diff --git a/you_providers/cookieUpdater.mjs b/you_providers/cookieUpdater.mjs index f07cb6d..48e8bd3 100644 --- a/you_providers/cookieUpdater.mjs +++ b/you_providers/cookieUpdater.mjs @@ -1,204 +1,423 @@ -import fs from "fs"; -import path from "path"; -import {fileURLToPath} from "url"; -import {Mutex} from "async-mutex"; - -const configMutex = new Mutex(); // 互斥锁 +import fs from 'fs'; +import path from 'path'; +import {Mutex} from 'async-mutex'; +import {detectBrowser} from './utils/browserDetector.mjs'; +import {createDirectoryIfNotExists} from './utils.mjs'; +import {fileURLToPath} from 'url'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); -const CONFIG_FILE_PATH = path.join(__dirname, "../config.mjs"); - -// 仅在 USE_MANUAL_LOGIN 为 false 且 ENABLE_AUTO_COOKIE_UPDATE 为 true 时生效 -const ENABLE_AUTO_COOKIE_UPDATE = process.env.ENABLE_AUTO_COOKIE_UPDATE === "true"; - -function unifyQuotesForJSON(str) { - // 正则匹配 `` `...` `` - let out = str.replace(/`([^`]*)`/g, (match, p1) => { - const safe = p1.replace(/"/g, '\\"'); - return `"${safe}"`; - }); - out = out.replace(/'([^']*)'/g, (match, p1) => { - const safe = p1.replace(/"/g, '\\"'); - return `"${safe}"`; - }); - - return out; +const isHeadless = process.env.HEADLESS_BROWSER === 'true' && process.env.USE_MANUAL_LOGIN !== 'true'; +let puppeteerModule; +let connect; +if (isHeadless === false) { + puppeteerModule = await import('puppeteer-real-browser'); + connect = puppeteerModule.connect; +} else { + puppeteerModule = await import('puppeteer-core'); } +// 会话自动释放时间（秒） +const SESSION_LOCK_TIMEOUT = parseInt(process.env.SESSION_LOCK_TIMEOUT || '0', 10); -/** - * cookies 解析出 DS 与 DSR - * @param {Array} cookies 获取到的 cookie 数组 - * @returns {{ ds?: string, dsr?: string }} - */ -function parseDSAndDSR(cookies) { - let dsValue, dsrValue; - for (const c of cookies) { - if (c.name === "DS") { - dsValue = c.value; - } else if (c.name === "DSR") { - dsrValue = c.value; +// 存储已达请求上限的账号(格式: "timestamp | username") +const cooldownFilePath = path.join(__dirname, 'cooldownAccounts.log'); + +// 冷却时长(默认24小时) +const COOLDOWN_DURATION = 24 * 60 * 60 * 1000; + +class SessionManager { + constructor(provider) { + this.provider = provider; + this.isCustomModeEnabled = process.env.USE_CUSTOM_MODE === 'true'; + this.isRotationEnabled = process.env.ENABLE_MODE_ROTATION === 'true'; + this.isHeadless = isHeadless; // 是否隐藏浏览器 + this.currentIndex = 0; + this.usernameList = []; // 缓存用户名列表 + this.browserInstances = []; // 浏览器实例数组 + this.browserMutex = new Mutex(); // 浏览器互斥锁 + this.browserIndex = 0; + this.sessionAutoUnlockTimers = {}; // 自动解锁计时器 + this.cooldownList = this.loadCooldownList(); // 加载并清理 cooldown 文件 + this.cleanupCooldownList(); + } + + setSessions(sessions) { + this.sessions = sessions; + this.usernameList = Object.keys(this.sessions); + + // 为每个 session 初始化相关属性 + for (const username in this.sessions) { + const session = this.sessions[username]; + session.locked = false; // 标记会话是否被锁定 + session.requestCount = 0; // 请求计数 + session.valid = true; // 标记会话是否有效 + session.mutex = new Mutex(); // 创建互斥锁 + if (session.currentMode === undefined) { + session.currentMode = this.isCustomModeEnabled ? 'custom' : 'default'; + } + if (!session.modeStatus) { + session.modeStatus = { + default: true, + custom: true, + }; + } + session.rotationEnabled = true; // 是否启用模式轮换 + session.switchCounter = 0; // 模式切换计数器 + session.requestsInCurrentMode = 0; // 当前模式下的请求次数 + session.lastDefaultThreshold = 0; // 上次默认模式阈值 + session.switchThreshold = this.provider.getRandomSwitchThreshold(session); + + // 记录请求次数 + session.youTotalRequests = 0; + // 权重 + if (typeof session.weight !== 'number') { + session.weight = 1; + } } } - return {ds: dsValue, dsr: dsrValue}; -} -/** - * 从 DS 中解析 email 字段 - * @param {string} dsToken DS cookie - * @returns {string|null} 返回 email或null - */ -function decodeEmailFromDs(dsToken) { - try { - const parts = dsToken.split("."); - if (parts.length < 2) return null; - const payload = JSON.parse(Buffer.from(parts[1], "base64").toString("utf-8")); - return payload.email || null; - } catch (err) { - return null; - } -} - -/** - * cookie 数组转换 "name=value; name=value" - * @param {Array} cookies - * @returns {string} - */ -function cookiesToStringAll(cookies) { - return cookies.map(c => `${c.name}=${c.value}`).join("; "); -} - -/** - * cookie 转换数组 - * 每个对象如 { name, value } - * @param {string} cookieStr - * @returns {Array} - */ -function parseCookieString(cookieStr) { - return cookieStr.split("; ").map(entry => { - const [name, value] = entry.split("=", 2); - return {name, value}; - }); -} - -/** - * 本地 configObj.sessions 查找与指定 email 匹配的 session， - * @param {object} configObj 解析后 config - * @param {string} email 匹配的邮箱 - * @returns {{ index: number, oldCookie: string, ds: string, dsr: string } | null} - */ -function findSessionByEmail(configObj, email) { - if (!Array.isArray(configObj.sessions)) return null; - for (let i = 0; i < configObj.sessions.length; i++) { - const cookieStr = configObj.sessions[i].cookie || ""; - const dsMatch = /DS=([^;\s]+)/.exec(cookieStr); - if (!dsMatch) continue; - const dsValue = dsMatch[1]; - const dsEmail = decodeEmailFromDs(dsValue); - if (dsEmail && dsEmail.toLowerCase() === email.toLowerCase()) { - const dsrMatch = /DSR=([^;\s]+)/.exec(cookieStr); - const dsrValue = dsrMatch ? dsrMatch[1] : ""; - return { - index: i, - oldCookie: cookieStr, - ds: dsValue, - dsr: dsrValue - }; - } - } - return null; -} - -/** - * config.mjs 中匹配相同 email 的 session，若 DS 或 DSR 有变化，则更新整个 cookie - * @param {import('puppeteer-core').Page} page - */ -export async function updateLocalConfigCookieByEmail(page) { - if (!ENABLE_AUTO_COOKIE_UPDATE || process.env.USE_MANUAL_LOGIN === "true") { - return; - } - // 尝试从 “https://you.com/api/instrumentation” 获取 cookie - let cookieStringFromInstrumentation = ""; - try { - const instrRequest = await page.waitForRequest( - req => req.url().includes("/api/instrumentation"), - {timeout: 5000} - ); - if (instrRequest) { - cookieStringFromInstrumentation = instrRequest.headers()["cookie"]; - } - } catch (err) { - } - - let allCookiesString = ""; - if (cookieStringFromInstrumentation) { - allCookiesString = cookieStringFromInstrumentation; - } else { - // 使用 page.cookies() 获取 - const cookies = await page.cookies("https://you.com"); - allCookiesString = cookiesToStringAll(cookies); - } - - const cookieArray = parseCookieString(allCookiesString); - const {ds: newDs, dsr: newDsr} = parseDSAndDSR(cookieArray); - if (!newDs) { - console.log("网页未找到 DS，跳过更新。"); - return; - } - const newEmail = decodeEmailFromDs(newDs); - if (!newEmail) { - console.log("[网页无法从 DS 解出 email，跳过更新。"); - return; - } - - // 互斥区 - await configMutex.runExclusive(async () => { + loadCooldownList() { try { - if (!fs.existsSync(CONFIG_FILE_PATH)) { - console.warn(`找不到 config.mjs: ${CONFIG_FILE_PATH}`); - return; + if (!fs.existsSync(cooldownFilePath)) { + fs.writeFileSync(cooldownFilePath, '', 'utf8'); + return []; } - const raw = fs.readFileSync(CONFIG_FILE_PATH, "utf8"); - // 去掉 export const config = - let jsonString = raw.replace(/^export const config\s*=\s*/, "").trim(); + const lines = fs.readFileSync(cooldownFilePath, 'utf8') + .split('\n') + .map(line => line.trim()) + .filter(line => line.length > 0); - jsonString = unifyQuotesForJSON(jsonString); - - const configObj = JSON.parse(jsonString); - - const found = findSessionByEmail(configObj, newEmail); - if (!found) { - console.log(`未能在 config 中找到 email=${newEmail} 的 session，跳过更新。`); - return; + const arr = []; + for (const line of lines) { + const parts = line.split('|').map(x => x.trim()); + if (parts.length === 2) { + const timestamp = parseInt(parts[0], 10); + const name = parts[1]; + if (!isNaN(timestamp) && name) { + arr.push({time: timestamp, username: name}); + } + } } - - if (found.ds === newDs && found.dsr === newDsr) { - console.log(`DS/DSR 未变化(email=${newEmail})，不更新。`); - return; - } - - configObj.sessions[found.index].cookie = allCookiesString; - - const newFileContent = "export const config = " + JSON.stringify(configObj, null, 4); - fs.writeFileSync(CONFIG_FILE_PATH, newFileContent, "utf8"); - - console.log(`Cookie已更新(email=${newEmail})`); + return arr; } catch (err) { - console.warn("Cookie更新过程出错:", err); + console.error(`读取 ${cooldownFilePath} 出错:`, err); + return []; } - }); + } + + saveCooldownList() { + try { + const lines = this.cooldownList.map(item => `${item.time} | ${item.username}`); + fs.writeFileSync(cooldownFilePath, lines.join('\n') + '\n', 'utf8'); + } catch (err) { + console.error(`写入 ${cooldownFilePath} 出错:`, err); + } + } + + // 清理过期(超过指定冷却时长) + cleanupCooldownList() { + const now = Date.now(); + let changed = false; + this.cooldownList = this.cooldownList.filter(item => { + const expired = (now - item.time) >= COOLDOWN_DURATION; + if (expired) changed = true; + return !expired; + }); + if (changed) { + this.saveCooldownList(); + } + } + + recordLimitedAccount(username) { + const now = Date.now(); + const already = this.cooldownList.find(x => x.username === username); + if (!already) { + this.cooldownList.push({time: now, username}); + this.saveCooldownList(); + console.log(`写入冷却列表：${new Date(now).toLocaleString()} | ${username}`); + } + } + + // 是否在冷却期(24小时内) + isInCooldown(username) { + this.cleanupCooldownList(); + return this.cooldownList.some(item => item.username === username); + } + + // 批量初始化浏览器实例 + async initBrowserInstancesInBatch() { + const browserCount = parseInt(process.env.BROWSER_INSTANCE_COUNT) || 1; + // 可以是 'chrome', 'edge', 或 'auto' + const browserPath = detectBrowser('auto'); + const sharedProfilePath = path.join(__dirname, 'browser_profiles'); + createDirectoryIfNotExists(sharedProfilePath); + + const tasks = []; + for (let i = 0; i < browserCount; i++) { + const browserId = `browser_${i}`; + const userDataDir = path.join(sharedProfilePath, browserId); + createDirectoryIfNotExists(userDataDir); + + tasks.push(this.launchSingleBrowser(browserId, userDataDir, browserPath)); + } + + // 并行执行 + const results = await Promise.all(tasks); + for (const instanceInfo of results) { + this.browserInstances.push(instanceInfo); + console.log(`创建浏览器实例: ${instanceInfo.id}`); + } + } + + async launchSingleBrowser(browserId, userDataDir, browserPath) { + let browser, page; + if (isHeadless === false) { + // 使用 puppeteer-real-browser + const response = await connect({ + headless: 'auto', + turnstile: true, + customConfig: { + userDataDir: userDataDir, + executablePath: browserPath, + args: [ + '--no-sandbox', + '--disable-setuid-sandbox', + '--remote-debugging-address=::', + ], + }, + }); + browser = response.browser; + page = response.page; + } else { + // 使用 puppeteer-core + browser = await puppeteerModule.launch({ + headless: this.isHeadless, + executablePath: browserPath, + userDataDir: userDataDir, + args: [ + '--no-sandbox', + '--disable-setuid-sandbox', + '--disable-gpu', + '--disable-dev-shm-usage', + '--remote-debugging-port=0', + ], + }); + page = await browser.newPage(); + } + + return { + id: browserId, + browser: browser, + page: page, + locked: false, + }; + } + + async getAvailableBrowser() { + return await this.browserMutex.runExclusive(async () => { + const totalBrowsers = this.browserInstances.length; + + for (let i = 0; i < totalBrowsers; i++) { + const index = (this.browserIndex + i) % totalBrowsers; + const browserInstance = this.browserInstances[index]; + + if (!browserInstance.locked) { + browserInstance.locked = true; + this.browserIndex = (index + 1) % totalBrowsers; + return browserInstance; + } + } + throw new Error('当前负载已饱和，请稍后再试(以达到最大并发)'); + }); + } + + async releaseBrowser(browserId) { + await this.browserMutex.runExclusive(async () => { + const browserInstance = this.browserInstances.find(b => b.id === browserId); + if (browserInstance) { + browserInstance.locked = false; + } + }); + } + + async getAvailableSessions() { + const allSessionsLocked = this.usernameList.every(username => this.sessions[username].locked); + if (allSessionsLocked) { + throw new Error('所有会话处于饱和状态，请稍后再试(无可用账号)'); + } + + // 收集所有valid && !locked && (不在冷却期) + let candidates = []; + for (const username of this.usernameList) { + const session = this.sessions[username]; + // 如果没被锁 并且 session.valid + if (session.valid && !session.locked) { + if (this.provider.enableRequestLimit && this.isInCooldown(username)) { + // console.log(`账号 ${username} 处于 24 小时冷却中，跳过`); + continue; + } + candidates.push(username); + } + } + + if (candidates.length === 0) { + throw new Error('没有可用的会话'); + } + + // 随机洗牌 + shuffleArray(candidates); + + // 加权抽签 + let weightSum = 0; + for (const uname of candidates) { + weightSum += this.sessions[uname].weight; + } + + // 生成随机 + const randValue = Math.floor(Math.random() * weightSum) + 1; + + // 遍历并扣减 + let cumulative = 0; + let selectedUsername = null; + for (const uname of candidates) { + cumulative += this.sessions[uname].weight; + if (randValue <= cumulative) { + selectedUsername = uname; + break; + } + } + + if (!selectedUsername) { + selectedUsername = candidates[0]; + } + + const selectedSession = this.sessions[selectedUsername]; + + // 再尝试锁定账号 + const result = await selectedSession.mutex.runExclusive(async () => { + if (selectedSession.locked) { + return null; + } + + // 判断是否可用 + if (selectedSession.modeStatus && selectedSession.modeStatus[selectedSession.currentMode]) { + // 锁定 + selectedSession.locked = true; + selectedSession.requestCount++; + + // 获取可用浏览器 + const browserInstance = await this.getAvailableBrowser(); + + // 启动自动解锁计时器 + if (SESSION_LOCK_TIMEOUT > 0) { + this.startAutoUnlockTimer(selectedUsername, browserInstance.id); + } + + return { + selectedUsername, + modeSwitched: false, + browserInstance + }; + } else if ( + this.isCustomModeEnabled && + this.isRotationEnabled && + this.provider && + typeof this.provider.switchMode === 'function' + ) { + console.warn(`尝试为账号 ${selectedUsername} 切换模式...`); + this.provider.switchMode(selectedSession); + selectedSession.rotationEnabled = false; + + if (selectedSession.modeStatus && selectedSession.modeStatus[selectedSession.currentMode]) { + selectedSession.locked = true; + selectedSession.requestCount++; + const browserInstance = await this.getAvailableBrowser(); + + if (SESSION_LOCK_TIMEOUT > 0) { + this.startAutoUnlockTimer(selectedUsername, browserInstance.id); + } + + return { + selectedUsername, + modeSwitched: true, + browserInstance + }; + } + } + + return null; + }); + + if (result) { + return result; + } else { + throw new Error('会话刚被占用或模式不可用!'); + } + } + + startAutoUnlockTimer(username, browserId) { + // 清除可能残留计时器 + if (this.sessionAutoUnlockTimers[username]) { + clearTimeout(this.sessionAutoUnlockTimers[username]); + } + const lockDurationMs = SESSION_LOCK_TIMEOUT * 1000; + + this.sessionAutoUnlockTimers[username] = setTimeout(async () => { + const session = this.sessions[username]; + if (session && session.locked) { + console.warn( + `会话 "${username}" 已自动解锁` + ); + + await session.mutex.runExclusive(async () => { + session.locked = false; + }); + + } + }, lockDurationMs); + } + + async releaseSession(username, browserId) { + const session = this.sessions[username]; + if (session) { + await session.mutex.runExclusive(() => { + session.locked = false; + }); + } + // 存在相应计时器清除 + if (this.sessionAutoUnlockTimers[username]) { + clearTimeout(this.sessionAutoUnlockTimers[username]); + delete this.sessionAutoUnlockTimers[username]; + } + + if (browserId) { + await this.releaseBrowser(browserId); + } + } + + // 返回会话 + // getBrowserInstances() { + // return this.browserInstances; + // } + + // 策略 + async getSessionByStrategy(strategy = 'round_robin') { + if (strategy === 'round_robin') { + return await this.getAvailableSessions(); + } + throw new Error(`未实现的策略: ${strategy}`); + } } /** - * 非阻塞 - * @param {import('puppeteer-core').Page} page + * Fisher–Yates 洗牌 */ -export function updateLocalConfigCookieByEmailNonBlocking(page) { - // 保证异步 - setImmediate(() => { - updateLocalConfigCookieByEmail(page).catch(err => - console.error("Cookie update error:", err) - ); - }); +function shuffleArray(array) { + for (let i = array.length - 1; i > 0; i--) { + const j = Math.floor(Math.random() * (i + 1)); + [array[i], array[j]] = [array[j], array[i]]; + } } + +export default SessionManager;