Google API Update (1.5.1) (#247)

* switch to google internal api

* A bit of tidying up

* A bit more of tidying up

* A bit more of tidying up

* Pre 1.5.1

- Add proxy exclusions
- Update ReadMe
- Update config Interface

---------

Co-authored-by: TheNetsky <56271887+TheNetsky@users.noreply.github.com>
This commit is contained in:
AariaX
2025-02-24 21:49:19 +07:00
committed by GitHub
parent 82a896e83f
commit 1102f2ca94
6 changed files with 98 additions and 64 deletions

View File

@@ -57,9 +57,11 @@ A basic docker `compose.yaml` is provided. Follow these steps to configure and r
| baseURL | MS Rewards page | `https://rewards.bing.com` |
| sessionPath | Path to where you want sessions/fingerprints to be stored | `sessions` (In ./browser/sessions) |
| headless | If the browser window should be visible be ran in the background | `false` (Browser is visible) |
| parallel | If you want mobile and desktop taks to run parallel or sequential| `true` |
| runOnZeroPoints | Run the rest of the script if 0 points can be earned | `false` (Will not run on 0 points) |
| clusters | Amount of instances ran on launch, 1 per account | `1` (Will run 1 account at the time) |
| saveFingerprint | Re-use the same fingerprint each time | `false` (Will generate a new fingerprint each time) |
| saveFingerprint.mobile | Re-use the same fingerprint each time | `false` (Will generate a new fingerprint each time) |
| saveFingerprint.desktop | Re-use the same fingerprint each time | `false` (Will generate a new fingerprint each time) |
| workers.doDailySet | Complete daily set items | `true` |
| workers.doMorePromotions | Complete promotional items | `true` |
| workers.doPunchCards | Complete punchcards | `true` |
@@ -67,17 +69,19 @@ A basic docker `compose.yaml` is provided. Follow these steps to configure and r
| workers.doMobileSearch | Complete daily mobile searches | `true` |
| workers.doDailyCheckIn | Complete daily check-in activity | `true` |
| workers.doReadToEarn | Complete read to earn activity | `true` |
| searchOnBingLocalQueries | Complete the activity "search on Bing" using the `queries.json` or fetched from this repo | `false` (Will fetch from this repo) |
| globalTimeout | The length before the action gets timeout | `30s` |
| searchSettings.useGeoLocaleQueries | Generate search queries based on your geo-location | `true` (Uses EN-US generated queries) |
| scrollRandomResults | Scroll randomly in search results | `true` |
| searchSettings.scrollRandomResults | Scroll randomly in search results | `true` |
| searchSettings.clickRandomResults | Visit random website from search result| `true` |
| searchSettings.searchDelay | Minimum and maximum time in miliseconds between search queries | `min: 1min` `max: 2min` |
| searchSettings.retryMobileSearchAmount | Keep retrying mobile searches for specified amount | `3` |
| logExcludeFunc | Functions to exclude out of the logs and webhooks | `SEARCH-CLOSE-TABS` |
| webhooklogExcludeFunc | Functions to exclude out of the webhooks log | `SEARCH-CLOSE-TABS` |
| webhookLogExcludeFunc | Functions to exclude out of the webhooks log | `SEARCH-CLOSE-TABS` |
| proxy.proxyGoogleTrends | Enable or disable proxying the request via set proxy | `true` (will be proxied) |
| proxy.proxyBingTerms | Enable or disable proxying the request via set proxy | `true` (will be proxied) |
| webhook.enabled | Enable or disable your set webhook | `false` |
| webhook.url | Your Discord webhook URL | `null` |
| cronStartTime | Scheduled script run-time, *only available for docker implementation* | `0 5,11 * * *` (5:00 am, 11:00 am daily) |
| | Run the script immediately when the Docker container starts | `true` |
## Features ##

View File

@@ -1,7 +1,7 @@
{
"baseURL": "https://rewards.bing.com",
"sessionPath": "sessions",
"headless": true,
"headless": false,
"parallel": true,
"runOnZeroPoints": false,
"clusters": 1,
@@ -33,11 +33,15 @@
"logExcludeFunc": [
"SEARCH-CLOSE-TABS"
],
"webhooklogExcludeFunc": [
"webhookLogExcludeFunc": [
"SEARCH-CLOSE-TABS"
],
"proxy": {
"proxyGoogleTrends": true,
"proxyBingTerms": true
},
"webhook": {
"enabled": false,
"url": ""
}
}
}

View File

@@ -4,9 +4,17 @@ import { platform } from 'os'
import { Workers } from '../Workers'
import { Counters, DashboardData } from '../../interface/DashboardData'
import { GoogleTrends } from '../../interface/GoogleDailyTrends'
import { GoogleSearch } from '../../interface/Search'
import { AxiosRequestConfig } from 'axios'
type GoogleTrendsResponse = [
string,
[
string,
...null[],
[string, ...string[]]
][]
];
export class Search extends Workers {
private bingHome = 'https://bing.com'
@@ -26,7 +34,7 @@ export class Search extends Workers {
}
// Generate search queries
let googleSearchQueries = await this.getGoogleTrends(data.userProfile.attributes.country, missingPoints)
let googleSearchQueries = await this.getGoogleTrends(this.bot.config.searchSettings.useGeoLocaleQueries ? data.userProfile.attributes.country : 'US')
googleSearchQueries = this.bot.utils.shuffleArray(googleSearchQueries)
// Deduplicate the search terms
@@ -203,49 +211,64 @@ export class Search extends Workers {
return await this.bot.browser.func.getSearchPoints()
}
private async getGoogleTrends(geoLocale: string, queryCount: number): Promise<GoogleSearch[]> {
private async getGoogleTrends(geoLocale: string = 'US'): Promise<GoogleSearch[]> {
const queryTerms: GoogleSearch[] = []
let i = 0
geoLocale = (this.bot.config.searchSettings.useGeoLocaleQueries && geoLocale.length === 2) ? geoLocale.toUpperCase() : 'US'
this.bot.log(this.bot.isMobile, 'SEARCH-GOOGLE-TRENDS', `Generating search queries, can take a while! | GeoLocale: ${geoLocale}`)
while (queryCount > queryTerms.length) {
i += 1
const date = new Date()
date.setDate(date.getDate() - i)
const formattedDate = this.formatDate(date)
try {
const request = {
url: `https://trends.google.com/trends/api/dailytrends?geo=${geoLocale}&hl=en&ed=${formattedDate}&ns=15`,
method: 'GET',
headers: {
'Content-Type': 'application/json'
}
}
const response = await this.bot.axios.request(request)
const data: GoogleTrends = JSON.parse((await response.data).slice(5))
for (const topic of data.default.trendingSearchesDays[0]?.trendingSearches ?? []) {
queryTerms.push({
topic: topic.title.query.toLowerCase(),
related: topic.relatedQueries.map(x => x.query.toLowerCase())
})
}
} catch (error) {
this.bot.log(this.bot.isMobile, 'SEARCH-GOOGLE-TRENDS', 'An error occurred:' + error, 'error')
break
try {
const request: AxiosRequestConfig = {
url: 'https://trends.google.com/_/TrendsUi/data/batchexecute',
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded;charset=UTF-8'
},
data: `f.req=[[[i0OFE,"[null, null, \\"${geoLocale}\\", 0, null, 48]"]]]`
}
const response = await this.bot.axios.request(request, this.bot.config.proxy.proxyGoogleTrends)
const rawText = response.data
const trendsData = this.extractJsonFromResponse(rawText)
if (!trendsData) {
throw this.bot.log(this.bot.isMobile, 'SEARCH-GOOGLE-TRENDS', 'Failed to parse Google Trends response', 'error')
}
const mappedTrendsData = trendsData.map(query => [query[0], query[9]!.slice(1)])
if (mappedTrendsData.length < 90) {
this.bot.log(this.bot.isMobile, 'SEARCH-GOOGLE-TRENDS', 'Insufficient search queries, falling back to US', 'warn')
return this.getGoogleTrends()
}
for (const [topic, relatedQueries] of mappedTrendsData) {
queryTerms.push({
topic: topic as string,
related: relatedQueries as string[]
})
}
} catch (error) {
this.bot.log(this.bot.isMobile, 'SEARCH-GOOGLE-TRENDS', 'An error occurred:' + error, 'error')
}
return queryTerms
}
private extractJsonFromResponse(text: string): GoogleTrendsResponse[1] | null {
const lines = text.split('\n')
for (const line of lines) {
const trimmed = line.trim()
if (trimmed.startsWith('[') && trimmed.endsWith(']')) {
try {
return JSON.parse(JSON.parse(trimmed)[0][2])[1]
} catch {
continue
}
}
}
return null
}
private async getRelatedTerms(term: string): Promise<string[]> {
try {
const request = {
@@ -256,7 +279,7 @@ export class Search extends Workers {
}
}
const response = await this.bot.axios.request(request)
const response = await this.bot.axios.request(request, this.bot.config.proxy.proxyBingTerms)
return response.data[1] as string[]
} catch (error) {
@@ -266,14 +289,6 @@ export class Search extends Workers {
return []
}
private formatDate(date: Date): string {
const year = date.getFullYear()
const month = String(date.getMonth() + 1).padStart(2, '0')
const day = String(date.getDate()).padStart(2, '0')
return `${year}${month}${day}`
}
private async randomScroll(page: Page) {
try {
const viewportHeight = await page.evaluate(() => window.innerHeight)

View File

@@ -5,14 +5,15 @@ export interface Config {
parallel: boolean;
runOnZeroPoints: boolean;
clusters: number;
workers: Workers;
saveFingerprint: ConfigSaveFingerprint;
workers: ConfigWorkers;
searchOnBingLocalQueries: boolean;
globalTimeout: number | string;
searchSettings: SearchSettings;
webhook: Webhook;
searchSettings: ConfigSearchSettings;
logExcludeFunc: string[];
webhooklogExcludeFunc: string[];
saveFingerprint: ConfigSaveFingerprint;
webhookLogExcludeFunc: string[];
proxy: ConfigProxy;
webhook: ConfigWebhook;
}
export interface ConfigSaveFingerprint {
@@ -20,25 +21,30 @@ export interface ConfigSaveFingerprint {
desktop: boolean;
}
export interface SearchSettings {
export interface ConfigSearchSettings {
useGeoLocaleQueries: boolean;
scrollRandomResults: boolean;
clickRandomResults: boolean;
searchDelay: SearchDelay;
searchDelay: ConfigSearchDelay;
retryMobileSearchAmount: number;
}
export interface SearchDelay {
export interface ConfigSearchDelay {
min: number | string;
max: number | string;
}
export interface Webhook {
export interface ConfigWebhook {
enabled: boolean;
url: string;
}
export interface Workers {
export interface ConfigProxy {
proxyGoogleTrends: boolean;
proxyBingTerms: boolean;
}
export interface ConfigWorkers {
doDailySet: boolean;
doMorePromotions: boolean;
doPunchCards: boolean;

View File

@@ -36,7 +36,12 @@ class AxiosClient {
}
// Generic method to make any Axios request
public async request(config: AxiosRequestConfig): Promise<AxiosResponse> {
public async request(config: AxiosRequestConfig, bypassProxy = false): Promise<AxiosResponse> {
if (bypassProxy) {
const bypassInstance = axios.create()
return bypassInstance.request(config)
}
return this.instance.request(config)
}
}

View File

@@ -19,7 +19,7 @@ export function log(isMobile: boolean | 'main', title: string, message: string,
const cleanStr = `[${currentTime}] [PID: ${process.pid}] [${type.toUpperCase()}] ${platformText} [${title}] ${message}`
// Send the clean string to the Webhook
if (!configData.webhooklogExcludeFunc.some(x => x.toLowerCase() === title.toLowerCase())) {
if (!configData.webhookLogExcludeFunc.some(x => x.toLowerCase() === title.toLowerCase())) {
Webhook(configData, cleanStr)
}