import { Page } from 'playwright' import axios from 'axios' import { Workers } from '../Workers' import { Counters, DashboardData } from '../../interface/DashboardData' import { GoogleTrends } from '../../interface/GoogleDailyTrends' import { GoogleSearch } from '../../interface/Search' export class Search extends Workers { private searchPageURL = 'https://bing.com' public async doSearch(page: Page, data: DashboardData) { this.bot.log('SEARCH-BING', 'Starting bing searches') page = await this.bot.browser.utils.getLatestTab(page) let searchCounters: Counters = await this.bot.browser.func.getSearchPoints() let missingPoints = this.calculatePoints(searchCounters) if (missingPoints === 0) { this.bot.log('SEARCH-BING', `Bing searches for ${this.bot.isMobile ? 'MOBILE' : 'DESKTOP'} have already been completed`) return } // Generate search queries let googleSearchQueries = await this.getGoogleTrends(data.userProfile.attributes.country, missingPoints) googleSearchQueries = this.bot.utils.shuffleArray(googleSearchQueries) // Deduplicate the search terms googleSearchQueries = [...new Set(googleSearchQueries)] // Go to bing await page.goto(this.searchPageURL) let maxLoop = 0 // If the loop hits 10 this when not gaining any points, we're assuming it's stuck. If it ddoesn't continue after 5 more searches with alternative queries, abort search const queries: string[] = [] // Mobile search doesn't seem to like related queries? googleSearchQueries.forEach(x => { this.bot.isMobile ? queries.push(x.topic) : queries.push(x.topic, ...x.related) }) // Loop over Google search queries for (let i = 0; i < queries.length; i++) { const query = queries[i] as string this.bot.log('SEARCH-BING', `${missingPoints} Points Remaining | Query: ${query} | Mobile: ${this.bot.isMobile}`) searchCounters = await this.bingSearch(page, query) const newMissingPoints = this.calculatePoints(searchCounters) // If the new point amount is the same as before if (newMissingPoints == missingPoints) { maxLoop++ // Add to max loop } else { // There has been a change in points maxLoop = 0 // Reset the loop } missingPoints = newMissingPoints if (missingPoints === 0) { break } // Only for mobile searches if (maxLoop > 3 && this.bot.isMobile) { this.bot.log('SEARCH-BING-MOBILE', 'Search didn\'t gain point for 3 iterations, likely bad User-Agent', 'warn') break } // If we didn't gain points for 10 iterations, assume it's stuck if (maxLoop > 10) { this.bot.log('SEARCH-BING', 'Search didn\'t gain point for 10 iterations aborting searches', 'warn') maxLoop = 0 // Reset to 0 so we can retry with related searches below break } } // Only for mobile searches if (missingPoints > 0 && this.bot.isMobile) { return } // If we still got remaining search queries, generate extra ones if (missingPoints > 0) { this.bot.log('SEARCH-BING', `Search completed but we're missing ${missingPoints} points, generating extra searches`) let i = 0 while (missingPoints > 0) { const query = googleSearchQueries[i++] as GoogleSearch // Get related search terms to the Google search queries const relatedTerms = await this.getRelatedTerms(query?.topic) if (relatedTerms.length > 3) { // Search for the first 2 related terms for (const term of relatedTerms.slice(1, 3)) { this.bot.log('SEARCH-BING-EXTRA', `${missingPoints} Points Remaining | Query: ${term} | Mobile: ${this.bot.isMobile}`) searchCounters = await this.bingSearch(page, query.topic) const newMissingPoints = this.calculatePoints(searchCounters) // If the new point amount is the same as before if (newMissingPoints == missingPoints) { maxLoop++ // Add to max loop } else { // There has been a change in points maxLoop = 0 // Reset the loop } missingPoints = newMissingPoints // If we satisfied the searches if (missingPoints === 0) { break } // Try 5 more times, then we tried a total of 15 times, fair to say it's stuck if (maxLoop > 5) { this.bot.log('SEARCH-BING-EXTRA', 'Search didn\'t gain point for 5 iterations aborting searches', 'warn') return } } } } } this.bot.log('SEARCH-BING', 'Completed searches') } private async bingSearch(searchPage: Page, query: string) { // Try a max of 5 times for (let i = 0; i < 5; i++) { try { const searchBar = '#sb_form_q' await searchPage.waitForSelector(searchBar, { state: 'attached', timeout: 10_000 }) await searchPage.click(searchBar) // Focus on the textarea await this.bot.utils.wait(500) await searchPage.keyboard.down('Control') await searchPage.keyboard.press('A') await searchPage.keyboard.press('Backspace') await searchPage.keyboard.up('Control') await searchPage.keyboard.type(query) await searchPage.keyboard.press('Enter') if (this.bot.config.searchSettings.scrollRandomResults) { await this.bot.utils.wait(2000) await this.randomScroll(searchPage) } if (this.bot.config.searchSettings.clickRandomResults) { await this.bot.utils.wait(2000) await this.clickRandomLink(searchPage) } await this.bot.utils.wait(Math.floor(this.bot.utils.randomNumber(this.bot.config.searchSettings.searchDelay.min, this.bot.config.searchSettings.searchDelay.max))) return await this.bot.browser.func.getSearchPoints() } catch (error) { if (i === 5) { this.bot.log('SEARCH-BING', 'Failed after 5 retries... An error occurred:' + error, 'error') break } this.bot.log('SEARCH-BING', 'Search failed, An error occurred:' + error, 'error') this.bot.log('SEARCH-BING', `Retrying search, attempt ${i}/5`, 'warn') // Reset the tabs const lastTab = await this.bot.browser.utils.getLatestTab(searchPage) await this.closeTabs(lastTab, this.searchPageURL) await this.bot.utils.wait(4000) } } this.bot.log('SEARCH-BING', 'Search failed after 5 retries, ending', 'error') return await this.bot.browser.func.getSearchPoints() } private async getGoogleTrends(geoLocale: string, queryCount: number): Promise { const queryTerms: GoogleSearch[] = [] let i = 0 geoLocale = (this.bot.config.searchSettings.useGeoLocaleQueries && geoLocale.length === 2) ? geoLocale.toUpperCase() : 'US' this.bot.log('SEARCH-GOOGLE-TRENDS', `Generating search queries, can take a while! | GeoLocale: ${geoLocale}`) while (queryCount > queryTerms.length) { i += 1 const date = new Date() date.setDate(date.getDate() - i) const formattedDate = this.formatDate(date) try { const request = { url: `https://trends.google.com/trends/api/dailytrends?geo=${geoLocale}&hl=en&ed=${formattedDate}&ns=15`, method: 'GET', headers: { 'Content-Type': 'application/json' } } const response = await axios(request) const data: GoogleTrends = JSON.parse((await response.data).slice(5)) for (const topic of data.default.trendingSearchesDays[0]?.trendingSearches ?? []) { queryTerms.push({ topic: topic.title.query.toLowerCase(), related: topic.relatedQueries.map(x => x.query.toLowerCase()) }) } } catch (error) { this.bot.log('SEARCH-GOOGLE-TRENDS', 'An error occurred:' + error, 'error') } } return queryTerms } private async getRelatedTerms(term: string): Promise { try { const request = { url: `https://api.bing.com/osjson.aspx?query=${term}`, method: 'GET', headers: { 'Content-Type': 'application/json' } } const response = await axios(request) return response.data[1] as string[] } catch (error) { this.bot.log('SEARCH-BING-RELTATED', 'An error occurred:' + error, 'error') } return [] } private formatDate(date: Date): string { const year = date.getFullYear() const month = String(date.getMonth() + 1).padStart(2, '0') const day = String(date.getDate()).padStart(2, '0') return `${year}${month}${day}` } private async randomScroll(page: Page) { try { // Press the arrow down key to scroll for (let i = 0; i < this.bot.utils.randomNumber(5, 600); i++) { await page.keyboard.press('ArrowDown') } } catch (error) { this.bot.log('SEARCH-RANDOM-SCROLL', 'An error occurred:' + error, 'error') } } private async clickRandomLink(page: Page) { try { const searchListingURL = new URL(page.url()) // Get searchPage info before clicking await page.click('#b_results .b_algo h2').catch(() => { }) // Since we don't really care if it did it or not // Will get current tab if no new one is created let lastTab = await this.bot.browser.utils.getLatestTab(page) // Let website load, if it doesn't load within 5 sec. exit regardless await this.bot.utils.wait(5000) let lastTabURL = new URL(lastTab.url()) // Get new tab info // Check if the URL is different from the original one, don't loop more than 5 times. let i = 0 while (lastTabURL.href !== searchListingURL.href && i < 5) { await this.closeTabs(lastTab, searchListingURL.href) // End of loop, refresh lastPage lastTab = await this.bot.browser.utils.getLatestTab(page) // Finally update the lastTab var again lastTabURL = new URL(lastTab.url()) // Get new tab info i++ } } catch (error) { this.bot.log('SEARCH-RANDOM-CLICK', 'An error occurred:' + error, 'error') } } private async closeTabs(lastTab: Page, url: string) { const browser = lastTab.context() const tabs = browser.pages() // If more than 3 tabs are open, close the last tab if (tabs.length > 2) { await lastTab.close() // If only 1 tab is open, open a new one to search in } else if (tabs.length === 1) { const newPage = await browser.newPage() await newPage.goto(url) // Else go back one page } else { await lastTab.goBack() } } private calculatePoints(counters: Counters) { const mobileData = counters.mobileSearch?.[0] // Mobile searches const genericData = counters.pcSearch?.[0] // Normal searches const edgeData = counters.pcSearch?.[1] // Edge searches const missingPoints = (this.bot.isMobile && mobileData) ? mobileData.pointProgressMax - mobileData.pointProgress : (edgeData ? edgeData.pointProgressMax - edgeData.pointProgress : 0) + (genericData ? genericData.pointProgressMax - genericData.pointProgress : 0) return missingPoints } }