Refactor page reload and dashboard extraction logic; consolidate dismissal methods in BrowserUtil; enhance Login TOTP handling; streamline environment variable parsing in scheduler

This commit is contained in:
2025-11-03 16:29:38 +01:00
parent 1531cff4a3
commit 579c8a4dcd
5 changed files with 144 additions and 178 deletions

View File

@@ -117,133 +117,38 @@ export default class BrowserFunc {
await this.goHome(target)
}
let lastError: unknown = null
for (let attempt = 1; attempt <= 2; attempt++) {
try {
// Reload the page to get new data
await target.reload({ waitUntil: 'domcontentloaded' })
lastError = null
break
} catch (re) {
lastError = re
const msg = (re instanceof Error ? re.message : String(re))
this.bot.log(this.bot.isMobile, 'GET-DASHBOARD-DATA', `Reload failed attempt ${attempt}: ${msg}`, 'warn')
// If page/context closed => bail early after first retry
if (msg.includes('has been closed')) {
if (attempt === 1) {
this.bot.log(this.bot.isMobile, 'GET-DASHBOARD-DATA', 'Page appears closed; trying one navigation fallback', 'warn')
try {
await this.goHome(target)
} catch {/* ignore */}
} else {
break
}
}
if (attempt === 2) {
await this.bot.utils.wait(1000)
}
}
}
// If reload failed after all attempts, throw the last error
if (lastError) {
throw lastError
}
// Wait a bit longer for scripts to load, especially on mobile
await this.bot.utils.wait(this.bot.isMobile ? TIMEOUTS.LONG : TIMEOUTS.MEDIUM)
// Reload with retry
await this.reloadPageWithRetry(target, 2)
// Wait for the more-activities element to ensure page is fully loaded
await target.waitForSelector(SELECTORS.MORE_ACTIVITIES, { timeout: TIMEOUTS.DASHBOARD_WAIT }).catch(() => {
this.bot.log(this.bot.isMobile, 'GET-DASHBOARD-DATA', 'Activities element not found, continuing anyway', 'warn')
})
let scriptContent = await target.evaluate(() => {
const scripts = Array.from(document.querySelectorAll('script'))
// Try multiple patterns for better compatibility
const targetScript = scripts.find(script =>
script.innerText.includes('var dashboard') ||
script.innerText.includes('dashboard=') ||
script.innerText.includes('dashboard :')
)
return targetScript?.innerText ? targetScript.innerText : null
})
let scriptContent = await this.extractDashboardScript(target)
if (!scriptContent) {
this.bot.log(this.bot.isMobile, 'GET-DASHBOARD-DATA', 'Dashboard script not found on first try, attempting recovery', 'warn')
await this.bot.browser.utils.captureDiagnostics(target, 'dashboard-data-missing').catch((e) => {
this.bot.log(this.bot.isMobile, 'GET-DASHBOARD-DATA', `Failed to capture diagnostics: ${e}`, 'warn')
})
await this.bot.browser.utils.captureDiagnostics(target, 'dashboard-data-missing').catch(() => {})
// Force a navigation retry once before failing hard
try {
await this.goHome(target)
await target.waitForLoadState('domcontentloaded', { timeout: TIMEOUTS.VERY_LONG }).catch((e) => {
this.bot.log(this.bot.isMobile, 'GET-DASHBOARD-DATA', `Wait for load state failed: ${e}`, 'warn')
})
await this.bot.utils.wait(this.bot.isMobile ? TIMEOUTS.LONG : TIMEOUTS.MEDIUM)
} catch (e) {
this.bot.log(this.bot.isMobile, 'GET-DASHBOARD-DATA', `Recovery navigation failed: ${e}`, 'warn')
}
await this.goHome(target)
await target.waitForLoadState('domcontentloaded', { timeout: TIMEOUTS.VERY_LONG }).catch(() => {})
await this.bot.utils.wait(this.bot.isMobile ? TIMEOUTS.LONG : TIMEOUTS.MEDIUM)
const retryContent = await target.evaluate(() => {
const scripts = Array.from(document.querySelectorAll('script'))
const targetScript = scripts.find(script =>
script.innerText.includes('var dashboard') ||
script.innerText.includes('dashboard=') ||
script.innerText.includes('dashboard :')
)
return targetScript?.innerText ? targetScript.innerText : null
}).catch(()=>null)
scriptContent = await this.extractDashboardScript(target)
if (!retryContent) {
// Log additional debug info
const scriptsDebug = await target.evaluate(() => {
const scripts = Array.from(document.querySelectorAll('script'))
return scripts.map(s => s.innerText.substring(0, 100)).join(' | ')
}).catch(() => 'Unable to get script debug info')
this.bot.log(this.bot.isMobile, 'GET-DASHBOARD-DATA', `Available scripts preview: ${scriptsDebug}`, 'warn')
if (!scriptContent) {
this.bot.log(this.bot.isMobile, 'GET-DASHBOARD-DATA', 'Dashboard data not found within script', 'error')
throw new Error('Dashboard data not found within script - check page structure')
}
scriptContent = retryContent
}
// Extract the dashboard object from the script content
const dashboardData = await target.evaluate((scriptContent: string) => {
// Try multiple regex patterns for better compatibility
const patterns = [
/var dashboard = (\{.*?\});/s, // Original pattern
/var dashboard=(\{.*?\});/s, // No spaces
/var\s+dashboard\s*=\s*(\{.*?\});/s, // Flexible whitespace
/dashboard\s*=\s*(\{[\s\S]*?\});/ // More permissive
]
for (const regex of patterns) {
const match = regex.exec(scriptContent)
if (match && match[1]) {
try {
return JSON.parse(match[1])
} catch (e) {
// Try next pattern if JSON parsing fails
continue
}
}
}
return null
}, scriptContent)
const dashboardData = await this.parseDashboardFromScript(target, scriptContent)
if (!dashboardData) {
// Log a snippet of the script content for debugging
const scriptPreview = scriptContent.substring(0, 200)
this.bot.log(this.bot.isMobile, 'GET-DASHBOARD-DATA', `Script preview: ${scriptPreview}`, 'warn')
await this.bot.browser.utils.captureDiagnostics(target, 'dashboard-data-parse').catch((e) => {
this.bot.log(this.bot.isMobile, 'GET-DASHBOARD-DATA', `Failed to capture diagnostics: ${e}`, 'warn')
})
await this.bot.browser.utils.captureDiagnostics(target, 'dashboard-data-parse').catch(() => {})
this.bot.log(this.bot.isMobile, 'GET-DASHBOARD-DATA', 'Unable to parse dashboard script', 'error')
throw new Error('Unable to parse dashboard script - check diagnostics')
}
@@ -258,6 +163,81 @@ export default class BrowserFunc {
}
/**
* Reload page with retry logic
*/
private async reloadPageWithRetry(page: Page, maxAttempts: number): Promise<void> {
let lastError: unknown = null
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
try {
await page.reload({ waitUntil: 'domcontentloaded' })
await this.bot.utils.wait(this.bot.isMobile ? TIMEOUTS.LONG : TIMEOUTS.MEDIUM)
lastError = null
break
} catch (re) {
lastError = re
const msg = (re instanceof Error ? re.message : String(re))
this.bot.log(this.bot.isMobile, 'GET-DASHBOARD-DATA', `Reload failed attempt ${attempt}: ${msg}`, 'warn')
if (msg.includes('has been closed')) {
if (attempt === 1) {
this.bot.log(this.bot.isMobile, 'GET-DASHBOARD-DATA', 'Page appears closed; trying one navigation fallback', 'warn')
try { await this.goHome(page) } catch {/* ignore */}
} else {
break
}
}
if (attempt === maxAttempts) {
await this.bot.utils.wait(1000)
}
}
}
if (lastError) throw lastError
}
/**
* Extract dashboard script from page
*/
private async extractDashboardScript(page: Page): Promise<string | null> {
return await page.evaluate(() => {
const scripts = Array.from(document.querySelectorAll('script'))
const targetScript = scripts.find(script =>
script.innerText.includes('var dashboard') ||
script.innerText.includes('dashboard=') ||
script.innerText.includes('dashboard :')
)
return targetScript?.innerText ? targetScript.innerText : null
})
}
/**
* Parse dashboard object from script content
*/
private async parseDashboardFromScript(page: Page, scriptContent: string): Promise<DashboardData | null> {
return await page.evaluate((scriptContent: string) => {
const patterns = [
/var dashboard = (\{.*?\});/s,
/var dashboard=(\{.*?\});/s,
/var\s+dashboard\s*=\s*(\{.*?\});/s,
/dashboard\s*=\s*(\{[\s\S]*?\});/
]
for (const regex of patterns) {
const match = regex.exec(scriptContent)
if (match && match[1]) {
try {
return JSON.parse(match[1])
} catch (e) {
continue
}
}
}
return null
}, scriptContent)
}
/**
* Get search point counters
* @returns {Counters} Object of search counter data

View File

@@ -51,20 +51,17 @@ export default class BrowserUtil {
}
async tryDismissAllMessages(page: Page): Promise<void> {
const maxRounds = 3
for (let round = 0; round < maxRounds; round++) {
const dismissCount = await this.dismissRound(page)
if (dismissCount === 0) break
}
// Single-pass dismissal with all checks combined
await this.dismissAllInterruptors(page)
}
private async dismissRound(page: Page): Promise<number> {
let count = 0
count += await this.dismissStandardButtons(page)
count += await this.dismissOverlayButtons(page)
count += await this.dismissStreakDialog(page)
count += await this.dismissTermsUpdateDialog(page)
return count
private async dismissAllInterruptors(page: Page): Promise<void> {
await Promise.allSettled([
this.dismissStandardButtons(page),
this.dismissOverlayButtons(page),
this.dismissStreakDialog(page),
this.dismissTermsUpdateDialog(page)
])
}
private async dismissStandardButtons(page: Page): Promise<number> {

View File

@@ -521,13 +521,13 @@ export class Login {
// Step 1: expose alternative verification options if hidden
if (!acted) {
acted = await this.clickFirstVisibleSelector(page, this.totpAltOptionSelectors())
acted = await this.clickFirstVisibleSelector(page, Login.TOTP_SELECTORS.altOptions)
if (acted) await this.bot.utils.wait(900)
}
// Step 2: choose authenticator code option if available
if (!acted) {
acted = await this.clickFirstVisibleSelector(page, this.totpChallengeSelectors())
acted = await this.clickFirstVisibleSelector(page, Login.TOTP_SELECTORS.challenge)
if (acted) await this.bot.utils.wait(900)
}
@@ -591,9 +591,7 @@ export class Login {
'button:has-text("Enter a code from your authenticator app")',
'button:has-text("Use code from your authentication app")',
'button:has-text("Utiliser un code de vérification")',
'button:has-text("Utiliser un code de verification")',
'button:has-text("Entrer un code depuis votre application")',
'button:has-text("Entrez un code depuis votre application")',
'button:has-text("Entrez un code")',
'div[role="button"]:has-text("Use a verification code")',
'div[role="button"]:has-text("Enter a code")'
@@ -612,14 +610,10 @@ export class Login {
]
} as const
private totpInputSelectors(): readonly string[] { return Login.TOTP_SELECTORS.input }
private totpAltOptionSelectors(): readonly string[] { return Login.TOTP_SELECTORS.altOptions }
private totpChallengeSelectors(): readonly string[] { return Login.TOTP_SELECTORS.challenge }
// Locate the most likely authenticator input on the page using heuristics
private async findFirstTotpInput(page: Page): Promise<string | null> {
const headingHint = await this.detectTotpHeading(page)
for (const sel of this.totpInputSelectors()) {
for (const sel of Login.TOTP_SELECTORS.input) {
const loc = page.locator(sel).first()
if (await loc.isVisible().catch(() => false)) {
if (await this.isLikelyTotpInput(page, loc, sel, headingHint)) {

View File

@@ -55,9 +55,6 @@ export class MicrosoftRewardsBot {
private isDesktopRunning: boolean = false
private isMobileRunning: boolean = false
private pointsCanCollect: number = 0
private pointsInitial: number = 0
private activeWorkers: number
private browserFactory: Browser = new Browser(this)
private accounts: Account[]
@@ -796,20 +793,11 @@ export class MicrosoftRewardsBot {
const durationMs = accountEnd - accountStart
const totalCollected = desktopCollected + mobileCollected
// Calculate initial total: use the lower value between desktop and mobile initial points
// to avoid double-counting. In sequential mode, mobile starts with desktop's earned points.
// In parallel mode, both should start from the same baseline.
let initialTotal = 0
if (desktopInitial > 0 && mobileInitial > 0) {
// Both flows completed: take minimum (true baseline before any earning)
initialTotal = Math.min(desktopInitial, mobileInitial)
} else if (desktopInitial > 0) {
// Only desktop completed
initialTotal = desktopInitial
} else if (mobileInitial > 0) {
// Only mobile completed
initialTotal = mobileInitial
}
// Sequential mode: desktop runs first, mobile starts with desktop's end points
// Parallel mode: both start from same baseline, take minimum to avoid double-count
const initialTotal = this.config.parallel
? Math.min(desktopInitial || Infinity, mobileInitial || Infinity)
: (desktopInitial || mobileInitial || 0)
const endTotal = initialTotal + totalCollected
if (!banned.status) {
@@ -1003,28 +991,27 @@ export class MicrosoftRewardsBot {
const data = await this.browser.func.getDashboardData()
this.pointsInitial = data.userStatus.availablePoints
const initial = this.pointsInitial
const initial = data.userStatus.availablePoints
log(this.isMobile, 'MAIN-POINTS', `Current point count: ${this.pointsInitial}`)
log(this.isMobile, 'MAIN-POINTS', `Current point count: ${initial}`)
const browserEnarablePoints = await this.browser.func.getBrowserEarnablePoints()
// Tally all the desktop points
this.pointsCanCollect = browserEnarablePoints.dailySetPoints +
browserEnarablePoints.desktopSearchPoints
+ browserEnarablePoints.morePromotionsPoints
const pointsCanCollect = browserEnarablePoints.dailySetPoints +
browserEnarablePoints.desktopSearchPoints +
browserEnarablePoints.morePromotionsPoints
log(this.isMobile, 'MAIN-POINTS', `You can earn ${this.pointsCanCollect} points today`)
log(this.isMobile, 'MAIN-POINTS', `You can earn ${pointsCanCollect} points today`)
if (this.pointsCanCollect === 0) {
if (pointsCanCollect === 0) {
// Extra diagnostic breakdown so users know WHY it's zero
log(this.isMobile, 'MAIN-POINTS', `Breakdown (desktop): dailySet=${browserEnarablePoints.dailySetPoints} search=${browserEnarablePoints.desktopSearchPoints} promotions=${browserEnarablePoints.morePromotionsPoints}`)
log(this.isMobile, 'MAIN-POINTS', 'All desktop earnable buckets are zero. This usually means: tasks already completed today OR the daily reset has not happened yet for your time zone. If you still want to force run activities set execution.runOnZeroPoints=true in config.', 'log', 'yellow')
}
// If runOnZeroPoints is false and 0 points to earn, don't continue
if (!this.config.runOnZeroPoints && this.pointsCanCollect === 0) {
if (!this.config.runOnZeroPoints && pointsCanCollect === 0) {
log(this.isMobile, 'MAIN', 'No points to earn and "runOnZeroPoints" is set to "false", stopping!', 'log', 'yellow')
// Close desktop browser
@@ -1107,22 +1094,22 @@ export class MicrosoftRewardsBot {
await this.browser.func.goHome(this.homePage)
const data = await this.browser.func.getDashboardData()
const initialPoints = data.userStatus.availablePoints || this.pointsInitial || 0
const initialPoints = data.userStatus.availablePoints || 0
const browserEnarablePoints = await this.browser.func.getBrowserEarnablePoints()
const appEarnablePoints = await this.browser.func.getAppEarnablePoints(this.accessToken)
this.pointsCanCollect = browserEnarablePoints.mobileSearchPoints + appEarnablePoints.totalEarnablePoints
const pointsCanCollect = browserEnarablePoints.mobileSearchPoints + appEarnablePoints.totalEarnablePoints
log(this.isMobile, 'MAIN-POINTS', `You can earn ${this.pointsCanCollect} points today (Browser: ${browserEnarablePoints.mobileSearchPoints} points, App: ${appEarnablePoints.totalEarnablePoints} points)`)
log(this.isMobile, 'MAIN-POINTS', `You can earn ${pointsCanCollect} points today (Browser: ${browserEnarablePoints.mobileSearchPoints} points, App: ${appEarnablePoints.totalEarnablePoints} points)`)
if (this.pointsCanCollect === 0) {
if (pointsCanCollect === 0) {
log(this.isMobile, 'MAIN-POINTS', `Breakdown (mobile): browserSearch=${browserEnarablePoints.mobileSearchPoints} appTotal=${appEarnablePoints.totalEarnablePoints}`)
log(this.isMobile, 'MAIN-POINTS', 'All mobile earnable buckets are zero. Causes: mobile searches already maxed, daily set finished, or daily rollover not reached yet. You can force execution by setting execution.runOnZeroPoints=true.', 'log', 'yellow')
}
// If runOnZeroPoints is false and 0 points to earn, don't continue
if (!this.config.runOnZeroPoints && this.pointsCanCollect === 0) {
if (!this.config.runOnZeroPoints && pointsCanCollect === 0) {
log(this.isMobile, 'MAIN', 'No points to earn and "runOnZeroPoints" is set to "false", stopping!', 'log', 'yellow')
// Close mobile browser

View File

@@ -11,6 +11,30 @@ import type { Config } from './interface/Config'
type CronExpressionInfo = { expression: string; tz: string }
type DateTimeInstance = ReturnType<typeof DateTime.fromJSDate>
/**
* Parse environment variable as number with validation
*/
function parseEnvNumber(key: string, defaultValue: number, min: number, max: number): number {
const raw = process.env[key]
if (!raw) return defaultValue
const parsed = Number(raw)
if (isNaN(parsed)) {
void log('main', 'SCHEDULER', `Invalid ${key}="${raw}". Using default ${defaultValue}`, 'warn')
return defaultValue
}
if (parsed < min || parsed > max) {
void log('main', 'SCHEDULER', `${key}=${parsed} out of range [${min}, ${max}]. Using default ${defaultValue}`, 'warn')
return defaultValue
}
return parsed
}
/**
* Parse time from schedule config (supports 12h and 24h formats)
*/
function resolveTimeParts(schedule: Config['schedule'] | undefined): { tz: string; hour: number; minute: number } {
const tz = (schedule?.timeZone && IANAZone.isValidZone(schedule.timeZone)) ? schedule.timeZone : 'UTC'
@@ -118,29 +142,13 @@ async function runOnePass(): Promise<void> {
*/
async function runOnePassWithWatchdog(): Promise<void> {
// Heartbeat-aware watchdog configuration
// If a child is actively updating its heartbeat file, we allow it to run beyond the legacy timeout.
// Defaults are generous to allow first-day passes to finish searches with delays.
const parseEnvNumber = (key: string, fallback: number, min: number, max: number): number => {
const val = Number(process.env[key] || fallback)
if (isNaN(val) || val < min || val > max) {
void log('main', 'SCHEDULER', `Invalid ${key}="${process.env[key]}". Using default ${fallback}`, 'warn')
return fallback
}
return val
}
const staleHeartbeatMin = parseEnvNumber(
process.env.SCHEDULER_STALE_HEARTBEAT_MINUTES ? 'SCHEDULER_STALE_HEARTBEAT_MINUTES' : 'SCHEDULER_PASS_TIMEOUT_MINUTES',
30, 5, 1440
)
const staleHeartbeatMin = parseEnvNumber('SCHEDULER_STALE_HEARTBEAT_MINUTES', 30, 5, 1440)
const graceMin = parseEnvNumber('SCHEDULER_HEARTBEAT_GRACE_MINUTES', 15, 1, 120)
const hardcapMin = parseEnvNumber('SCHEDULER_PASS_HARDCAP_MINUTES', 480, 30, 1440)
const checkEveryMs = 60_000 // check once per minute
// Validate: stale should be >= grace
if (staleHeartbeatMin < graceMin) {
await log('main', 'SCHEDULER', `Warning: STALE_HEARTBEAT (${staleHeartbeatMin}m) < GRACE (${graceMin}m). Adjusting stale to ${graceMin}m`, 'warn')
}
const effectiveStale = Math.max(staleHeartbeatMin, graceMin)
// Fork per pass: safer because we can terminate a stuck child without killing the scheduler
const forkPerPass = String(process.env.SCHEDULER_FORK_PER_PASS || 'true').toLowerCase() !== 'false'
@@ -196,8 +204,8 @@ async function runOnePassWithWatchdog(): Promise<void> {
const st = fs.statSync(hbFile)
const mtimeMs = st.mtimeMs
const ageMin = Math.floor((now - mtimeMs) / 60000)
if (ageMin >= staleHeartbeatMin) {
log('main', 'SCHEDULER', `Heartbeat stale for ${ageMin}m (>=${staleHeartbeatMin}m). Terminating child...`, 'warn')
if (ageMin >= effectiveStale) {
log('main', 'SCHEDULER', `Heartbeat stale for ${ageMin}m (>=${effectiveStale}m). Terminating child...`, 'warn')
void killChild('SIGTERM')
if (killTimeout) clearTimeout(killTimeout)
killTimeout = setTimeout(() => { try { child.kill('SIGKILL') } catch { /* ignore */ } }, 10_000)