diff --git a/src/main/events/download-sources/helpers.ts b/src/main/events/download-sources/helpers.ts index 2e7489fd..edd3878e 100644 --- a/src/main/events/download-sources/helpers.ts +++ b/src/main/events/download-sources/helpers.ts @@ -192,83 +192,30 @@ export const addNewDownloads = async ( const batch = repacksSublevel.batch(); + // Get title hash mapping and perform matching in worker thread const titleHashMapping = await getTitleHashMapping(); - let hashMatchCount = 0; - let fuzzyMatchCount = 0; - let noMatchCount = 0; - for (const download of downloads) { - let objectIds: string[] = []; - let usedHashMatch = false; + const { GameMatcherWorkerManager } = await import("@main/services"); + const matchResult = await GameMatcherWorkerManager.matchDownloads( + downloads, + steamGames, + titleHashMapping + ); - const titleHash = hashTitle(download.title); - const steamIdsFromHash = titleHashMapping[titleHash]; - - if (steamIdsFromHash && steamIdsFromHash.length > 0) { - hashMatchCount++; - usedHashMatch = true; - - objectIds = steamIdsFromHash.map(String); - } - - if (!usedHashMatch) { - let gamesInSteam: FormattedSteamGame[] = []; - const formattedTitle = formatRepackName(download.title); - - if (formattedTitle && formattedTitle.length > 0) { - const [firstLetter] = formattedTitle; - const games = steamGames[firstLetter] || []; - - gamesInSteam = games.filter((game) => - formattedTitle.startsWith(game.formattedName) - ); - - if (gamesInSteam.length === 0) { - gamesInSteam = games.filter( - (game) => - formattedTitle.includes(game.formattedName) || - game.formattedName.includes(formattedTitle) - ); - } - - if (gamesInSteam.length === 0) { - for (const letter of Object.keys(steamGames)) { - const letterGames = steamGames[letter] || []; - const matches = letterGames.filter( - (game) => - formattedTitle.includes(game.formattedName) || - game.formattedName.includes(formattedTitle) - ); - if (matches.length > 0) { - gamesInSteam = matches; - break; - } - } - } - - if (gamesInSteam.length > 0) { - fuzzyMatchCount++; - objectIds = gamesInSteam.map((game) => String(game.id)); - } else { - noMatchCount++; - } - } else { - noMatchCount++; - } - } - - for (const id of objectIds) { + // Process matched results and write to database + for (const matchedDownload of matchResult.matchedDownloads) { + for (const id of matchedDownload.objectIds) { objectIdsOnSource.add(id); } const repack = { id: nextRepackId++, - objectIds: objectIds, - title: download.title, - uris: download.uris, - fileSize: download.fileSize, + objectIds: matchedDownload.objectIds, + title: matchedDownload.title, + uris: matchedDownload.uris, + fileSize: matchedDownload.fileSize, repacker: downloadSource.name, - uploadDate: download.uploadDate, + uploadDate: matchedDownload.uploadDate, downloadSourceId: downloadSource.id, createdAt: now, updatedAt: now, @@ -280,7 +227,7 @@ export const addNewDownloads = async ( await batch.write(); logger.info( - `Matching stats for ${downloadSource.name}: Hash=${hashMatchCount}, Fuzzy=${fuzzyMatchCount}, None=${noMatchCount}` + `Matching stats for ${downloadSource.name}: Hash=${matchResult.stats.hashMatchCount}, Fuzzy=${matchResult.stats.fuzzyMatchCount}, None=${matchResult.stats.noMatchCount}` ); const existingSource = await downloadSourcesSublevel.get( diff --git a/src/main/events/download-sources/sync-download-sources.ts b/src/main/events/download-sources/sync-download-sources.ts index 88861074..3bb78f22 100644 --- a/src/main/events/download-sources/sync-download-sources.ts +++ b/src/main/events/download-sources/sync-download-sources.ts @@ -31,20 +31,10 @@ const syncDownloadSources = async ( downloadSources.push(source); } - const existingRepacks: Array<{ - id: number; - title: string; - uris: string[]; - repacker: string; - fileSize: string | null; - objectIds: string[]; - uploadDate: Date | string | null; - downloadSourceId: number; - createdAt: Date; - updatedAt: Date; - }> = []; + // Use a Set for O(1) lookups instead of O(n) with array.some() + const existingRepackTitles = new Set(); for await (const [, repack] of repacksSublevel.iterator()) { - existingRepacks.push(repack); + existingRepackTitles.add(repack.title); } // Handle sources with missing fingerprints individually, don't delete all sources @@ -77,9 +67,9 @@ const syncDownloadSources = async ( const source = downloadSourceSchema.parse(response.data); const steamGames = await getSteamGames(); + // O(1) lookup instead of O(n) - massive performance improvement const repacks = source.downloads.filter( - (download) => - !existingRepacks.some((repack) => repack.title === download.title) + (download) => !existingRepackTitles.has(download.title) ); await downloadSourcesSublevel.put(`${downloadSource.id}`, { diff --git a/src/main/main.ts b/src/main/main.ts index 5eecb101..e9b6187c 100644 --- a/src/main/main.ts +++ b/src/main/main.ts @@ -17,6 +17,7 @@ import { Lock, DeckyPlugin, ResourceCache, + GameMatcherWorkerManager, } from "@main/services"; export const loadState = async () => { @@ -25,6 +26,9 @@ export const loadState = async () => { ResourceCache.initialize(); await ResourceCache.updateResourcesOnStartup(); + // Initialize game matcher worker thread + GameMatcherWorkerManager.initialize(); + const userPreferences = await db.get( levelKeys.userPreferences, { diff --git a/src/main/services/game-matcher-worker-manager.ts b/src/main/services/game-matcher-worker-manager.ts new file mode 100644 index 00000000..b5d306c7 --- /dev/null +++ b/src/main/services/game-matcher-worker-manager.ts @@ -0,0 +1,145 @@ +import { Worker } from "worker_threads"; +import workerPath from "../workers/game-matcher-worker?modulePath"; + +interface WorkerMessage { + id: string; + data: unknown; +} + +interface WorkerResponse { + id: string; + success: boolean; + result?: unknown; + error?: string; +} + +export type TitleHashMapping = Record; + +export type FormattedSteamGame = { + id: string; + name: string; + formattedName: string; +}; +export type FormattedSteamGamesByLetter = Record; + +interface DownloadToMatch { + title: string; + uris: string[]; + uploadDate: string; + fileSize: string; +} + +interface MatchedDownload { + title: string; + uris: string[]; + uploadDate: string; + fileSize: string; + objectIds: string[]; + usedHashMatch: boolean; +} + +interface MatchResponse { + matchedDownloads: MatchedDownload[]; + stats: { + hashMatchCount: number; + fuzzyMatchCount: number; + noMatchCount: number; + }; +} + +export class GameMatcherWorkerManager { + private static worker: Worker | null = null; + private static messageId = 0; + private static pendingMessages = new Map< + string, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + { resolve: (value: any) => void; reject: (error: Error) => void } + >(); + + public static initialize() { + if (this.worker) { + return; + } + + try { + console.log( + "[GameMatcherWorker] Initializing worker with path:", + workerPath + ); + + this.worker = new Worker(workerPath); + + this.worker.on("message", (response: WorkerResponse) => { + const pending = this.pendingMessages.get(response.id); + if (pending) { + if (response.success) { + pending.resolve(response.result); + } else { + pending.reject(new Error(response.error || "Unknown error")); + } + this.pendingMessages.delete(response.id); + } + }); + + this.worker.on("error", (error) => { + console.error("[GameMatcherWorker] Worker error:", error); + for (const [id, pending] of this.pendingMessages.entries()) { + pending.reject(error); + this.pendingMessages.delete(id); + } + }); + + this.worker.on("exit", (code) => { + if (code !== 0) { + console.error( + `[GameMatcherWorker] Worker stopped with exit code ${code}` + ); + } + this.worker = null; + for (const [id, pending] of this.pendingMessages.entries()) { + pending.reject(new Error("Worker exited unexpectedly")); + this.pendingMessages.delete(id); + } + }); + + console.log("[GameMatcherWorker] Worker initialized successfully"); + } catch (error) { + console.error("[GameMatcherWorker] Failed to initialize worker:", error); + throw error; + } + } + + private static sendMessage(data: unknown): Promise { + if (!this.worker) { + return Promise.reject(new Error("Worker not initialized")); + } + + const id = `msg_${++this.messageId}`; + const message: WorkerMessage = { id, data }; + + return new Promise((resolve, reject) => { + this.pendingMessages.set(id, { resolve, reject }); + this.worker!.postMessage(message); + }); + } + + public static async matchDownloads( + downloads: DownloadToMatch[], + steamGames: FormattedSteamGamesByLetter, + titleHashMapping: TitleHashMapping + ): Promise { + return this.sendMessage({ + downloads, + steamGames, + titleHashMapping, + }); + } + + public static terminate() { + if (this.worker) { + this.worker.terminate(); + this.worker = null; + this.pendingMessages.clear(); + } + } +} diff --git a/src/main/services/index.ts b/src/main/services/index.ts index c98f09e1..0853859f 100644 --- a/src/main/services/index.ts +++ b/src/main/services/index.ts @@ -19,3 +19,4 @@ export * from "./wine"; export * from "./lock"; export * from "./decky-plugin"; export * from "./resource-cache"; +export * from "./game-matcher-worker-manager"; diff --git a/src/main/workers/game-matcher-worker.ts b/src/main/workers/game-matcher-worker.ts new file mode 100644 index 00000000..4930ada0 --- /dev/null +++ b/src/main/workers/game-matcher-worker.ts @@ -0,0 +1,158 @@ +import { parentPort } from "worker_threads"; +import crypto from "node:crypto"; + +export type TitleHashMapping = Record; + +export type FormattedSteamGame = { + id: string; + name: string; + formattedName: string; +}; +export type FormattedSteamGamesByLetter = Record; + +interface DownloadToMatch { + title: string; + uris: string[]; + uploadDate: string; + fileSize: string; +} + +interface MatchedDownload { + title: string; + uris: string[]; + uploadDate: string; + fileSize: string; + objectIds: string[]; + usedHashMatch: boolean; +} + +interface MatchRequest { + downloads: DownloadToMatch[]; + steamGames: FormattedSteamGamesByLetter; + titleHashMapping: TitleHashMapping; +} + +interface MatchResponse { + matchedDownloads: MatchedDownload[]; + stats: { + hashMatchCount: number; + fuzzyMatchCount: number; + noMatchCount: number; + }; +} + +const hashTitle = (title: string): string => { + return crypto.createHash("sha256").update(title).digest("hex"); +}; + +const formatName = (name: string) => { + return name + .normalize("NFD") + .replaceAll(/[\u0300-\u036f]/g, "") + .toLowerCase() + .replaceAll(/[^a-z0-9]/g, ""); +}; + +const formatRepackName = (name: string) => { + return formatName(name.replace("[DL]", "")); +}; + +const matchDownloads = (request: MatchRequest): MatchResponse => { + const { downloads, steamGames, titleHashMapping } = request; + const matchedDownloads: MatchedDownload[] = []; + + let hashMatchCount = 0; + let fuzzyMatchCount = 0; + let noMatchCount = 0; + + for (const download of downloads) { + let objectIds: string[] = []; + let usedHashMatch = false; + + const titleHash = hashTitle(download.title); + const steamIdsFromHash = titleHashMapping[titleHash]; + + if (steamIdsFromHash && steamIdsFromHash.length > 0) { + hashMatchCount++; + usedHashMatch = true; + objectIds = steamIdsFromHash.map(String); + } + + if (!usedHashMatch) { + let gamesInSteam: FormattedSteamGame[] = []; + const formattedTitle = formatRepackName(download.title); + + if (formattedTitle && formattedTitle.length > 0) { + const [firstLetter] = formattedTitle; + const games = steamGames[firstLetter] || []; + + gamesInSteam = games.filter((game) => + formattedTitle.startsWith(game.formattedName) + ); + + if (gamesInSteam.length === 0) { + gamesInSteam = games.filter( + (game) => + formattedTitle.includes(game.formattedName) || + game.formattedName.includes(formattedTitle) + ); + } + + if (gamesInSteam.length === 0) { + for (const letter of Object.keys(steamGames)) { + const letterGames = steamGames[letter] || []; + const matches = letterGames.filter( + (game) => + formattedTitle.includes(game.formattedName) || + game.formattedName.includes(formattedTitle) + ); + if (matches.length > 0) { + gamesInSteam = matches; + break; + } + } + } + + if (gamesInSteam.length > 0) { + fuzzyMatchCount++; + objectIds = gamesInSteam.map((game) => String(game.id)); + } else { + noMatchCount++; + } + } else { + noMatchCount++; + } + } + + matchedDownloads.push({ + ...download, + objectIds, + usedHashMatch, + }); + } + + return { + matchedDownloads, + stats: { + hashMatchCount, + fuzzyMatchCount, + noMatchCount, + }, + }; +}; + +// Message handler +if (parentPort) { + parentPort.on("message", (message: { id: string; data: MatchRequest }) => { + try { + const result = matchDownloads(message.data); + parentPort!.postMessage({ id: message.id, success: true, result }); + } catch (error) { + parentPort!.postMessage({ + id: message.id, + success: false, + error: error instanceof Error ? error.message : String(error), + }); + } + }); +}