diff --git a/src/main/events/download-sources/helpers.ts b/src/main/events/download-sources/helpers.ts index 7bd8023c..b0c6841d 100644 --- a/src/main/events/download-sources/helpers.ts +++ b/src/main/events/download-sources/helpers.ts @@ -16,6 +16,8 @@ const downloadSourceSchema = z.object({ }); type SteamGamesByLetter = Record; +type FormattedSteamGame = { id: string; name: string; formattedName: string }; +type FormattedSteamGamesByLetter = Record; const formatName = (name: string) => { return name @@ -29,8 +31,48 @@ const formatRepackName = (name: string) => { return formatName(name.replace("[DL]", "")); }; +interface DownloadSource { + id: number; + url: string; + name: string; + etag: string | null; + status: number; + downloadCount: number; + objectIds: string[]; + fingerprint?: string; + createdAt: Date; + updatedAt: Date; +} + +let downloadSourcesCache: Map | null = null; +let downloadSourcesCacheTime = 0; +const CACHE_TTL = 5000; + +const getDownloadSourcesMap = async (): Promise< + Map +> => { + const now = Date.now(); + if (downloadSourcesCache && now - downloadSourcesCacheTime < CACHE_TTL) { + return downloadSourcesCache; + } + + const map = new Map(); + for await (const [key, source] of downloadSourcesSublevel.iterator()) { + map.set(key, source); + } + + downloadSourcesCache = map; + downloadSourcesCacheTime = now; + return map; +}; + +export const invalidateDownloadSourcesCache = () => { + downloadSourcesCache = null; +}; + export const checkUrlExists = async (url: string): Promise => { - for await (const [, source] of downloadSourcesSublevel.iterator()) { + const sources = await getDownloadSourcesMap(); + for (const source of sources.values()) { if (source.url === url) { return true; } @@ -38,12 +80,31 @@ export const checkUrlExists = async (url: string): Promise => { return false; }; -const getSteamGames = async () => { +let steamGamesCache: FormattedSteamGamesByLetter | null = null; +let steamGamesCacheTime = 0; +const STEAM_GAMES_CACHE_TTL = 300000; + +const getSteamGames = async (): Promise => { + const now = Date.now(); + if (steamGamesCache && now - steamGamesCacheTime < STEAM_GAMES_CACHE_TTL) { + return steamGamesCache; + } + const response = await axios.get( `${import.meta.env.MAIN_VITE_EXTERNAL_RESOURCES_URL}/steam-games-by-letter.json` ); - return response.data; + const formattedData: FormattedSteamGamesByLetter = {}; + for (const [letter, games] of Object.entries(response.data)) { + formattedData[letter] = games.map((game) => ({ + ...game, + formattedName: formatName(game.name), + })); + } + + steamGamesCache = formattedData; + steamGamesCacheTime = now; + return formattedData; }; type SublevelIterator = AsyncIterable<[string, { id: number }]>; @@ -52,33 +113,61 @@ interface SublevelWithId { iterator: () => SublevelIterator; } +let maxRepackId: number | null = null; +let maxDownloadSourceId: number | null = null; + const getNextId = async (sublevel: SublevelWithId): Promise => { + const isRepackSublevel = sublevel === repacksSublevel; + const isDownloadSourceSublevel = sublevel === downloadSourcesSublevel; + + if (isRepackSublevel && maxRepackId !== null) { + return ++maxRepackId; + } + + if (isDownloadSourceSublevel && maxDownloadSourceId !== null) { + return ++maxDownloadSourceId; + } + let maxId = 0; for await (const [, value] of sublevel.iterator()) { if (value.id > maxId) { maxId = value.id; } } + + if (isRepackSublevel) { + maxRepackId = maxId; + } else if (isDownloadSourceSublevel) { + maxDownloadSourceId = maxId; + } + return maxId + 1; }; +export const invalidateIdCaches = () => { + maxRepackId = null; + maxDownloadSourceId = null; +}; + const addNewDownloads = async ( downloadSource: { id: number; name: string }, downloads: z.infer["downloads"], - steamGames: SteamGamesByLetter + steamGames: FormattedSteamGamesByLetter ) => { const now = new Date(); const objectIdsOnSource = new Set(); let nextRepackId = await getNextId(repacksSublevel); + const batch = repacksSublevel.batch(); + for (const download of downloads) { const formattedTitle = formatRepackName(download.title); const [firstLetter] = formattedTitle; const games = steamGames[firstLetter] || []; const gamesInSteam = games.filter((game) => - formattedTitle.startsWith(formatName(game.name)) + formattedTitle.startsWith(game.formattedName) ); if (gamesInSteam.length === 0) continue; @@ -100,9 +189,11 @@ const addNewDownloads = async ( updatedAt: now, }; - await repacksSublevel.put(`${repack.id}`, repack); + batch.put(`${repack.id}`, repack); } + await batch.write(); + const existingSource = await downloadSourcesSublevel.get( `${downloadSource.id}` ); @@ -134,14 +225,6 @@ export const importDownloadSourceToLocal = async ( const now = new Date(); - const urlExistsBeforeInsert = await checkUrlExists(url); - if (urlExistsBeforeInsert) { - if (throwOnDuplicate) { - throw new Error("Download source with this URL already exists"); - } - return null; - } - const nextId = await getNextId(downloadSourcesSublevel); const downloadSource = { @@ -158,6 +241,8 @@ export const importDownloadSourceToLocal = async ( await downloadSourcesSublevel.put(`${downloadSource.id}`, downloadSource); + invalidateDownloadSourcesCache(); + const objectIds = await addNewDownloads( downloadSource, response.data.downloads, diff --git a/src/main/events/download-sources/sync-download-sources-from-api.ts b/src/main/events/download-sources/sync-download-sources-from-api.ts index 21b5a097..23769839 100644 --- a/src/main/events/download-sources/sync-download-sources-from-api.ts +++ b/src/main/events/download-sources/sync-download-sources-from-api.ts @@ -1,6 +1,5 @@ import { HydraApi } from "@main/services"; -import { downloadSourcesSublevel } from "@main/level"; -import { importDownloadSourceToLocal } from "./helpers"; +import { importDownloadSourceToLocal, checkUrlExists } from "./helpers"; export const syncDownloadSourcesFromApi = async () => { try { @@ -8,15 +7,9 @@ export const syncDownloadSourcesFromApi = async () => { { url: string; createdAt: string; updatedAt: string }[] >("/profile/download-sources"); - const localSources: { url: string }[] = []; - for await (const [, source] of downloadSourcesSublevel.iterator()) { - localSources.push(source); - } - - const localUrls = new Set(localSources.map((s) => s.url)); - for (const apiSource of apiSources) { - if (!localUrls.has(apiSource.url)) { + const exists = await checkUrlExists(apiSource.url); + if (!exists) { await importDownloadSourceToLocal(apiSource.url, false); } } diff --git a/src/main/events/download-sources/sync-download-sources.ts b/src/main/events/download-sources/sync-download-sources.ts index 8611aca9..19ab8079 100644 --- a/src/main/events/download-sources/sync-download-sources.ts +++ b/src/main/events/download-sources/sync-download-sources.ts @@ -3,6 +3,11 @@ import axios, { AxiosError } from "axios"; import { z } from "zod"; import { downloadSourcesSublevel, repacksSublevel } from "@main/level"; import { DownloadSourceStatus } from "@shared"; +import { + checkUrlExists, + invalidateDownloadSourcesCache, + invalidateIdCaches, +} from "./helpers"; const downloadSourceSchema = z.object({ name: z.string().max(255), @@ -17,6 +22,8 @@ const downloadSourceSchema = z.object({ }); type SteamGamesByLetter = Record; +type FormattedSteamGame = { id: string; name: string; formattedName: string }; +type FormattedSteamGamesByLetter = Record; const formatName = (name: string) => { return name @@ -30,21 +37,31 @@ const formatRepackName = (name: string) => { return formatName(name.replace("[DL]", "")); }; -const checkUrlExists = async (url: string): Promise => { - for await (const [, source] of downloadSourcesSublevel.iterator()) { - if (source.url === url) { - return true; - } - } - return false; -}; +let steamGamesCache: FormattedSteamGamesByLetter | null = null; +let steamGamesCacheTime = 0; +const STEAM_GAMES_CACHE_TTL = 300000; + +const getSteamGames = async (): Promise => { + const now = Date.now(); + if (steamGamesCache && now - steamGamesCacheTime < STEAM_GAMES_CACHE_TTL) { + return steamGamesCache; + } -const getSteamGames = async () => { const response = await axios.get( `${import.meta.env.MAIN_VITE_EXTERNAL_RESOURCES_URL}/steam-games-by-letter.json` ); - return response.data; + const formattedData: FormattedSteamGamesByLetter = {}; + for (const [letter, games] of Object.entries(response.data)) { + formattedData[letter] = games.map((game) => ({ + ...game, + formattedName: formatName(game.name), + })); + } + + steamGamesCache = formattedData; + steamGamesCacheTime = now; + return formattedData; }; type SublevelIterator = AsyncIterable<[string, { id: number }]>; @@ -53,33 +70,56 @@ interface SublevelWithId { iterator: () => SublevelIterator; } +let maxRepackId: number | null = null; +let maxDownloadSourceId: number | null = null; + const getNextId = async (sublevel: SublevelWithId): Promise => { + const isRepackSublevel = sublevel === repacksSublevel; + const isDownloadSourceSublevel = sublevel === downloadSourcesSublevel; + + if (isRepackSublevel && maxRepackId !== null) { + return ++maxRepackId; + } + + if (isDownloadSourceSublevel && maxDownloadSourceId !== null) { + return ++maxDownloadSourceId; + } + let maxId = 0; for await (const [, value] of sublevel.iterator()) { if (value.id > maxId) { maxId = value.id; } } + + if (isRepackSublevel) { + maxRepackId = maxId; + } else if (isDownloadSourceSublevel) { + maxDownloadSourceId = maxId; + } + return maxId + 1; }; const addNewDownloads = async ( downloadSource: { id: number; name: string }, downloads: z.infer["downloads"], - steamGames: SteamGamesByLetter + steamGames: FormattedSteamGamesByLetter ) => { const now = new Date(); const objectIdsOnSource = new Set(); let nextRepackId = await getNextId(repacksSublevel); + const batch = repacksSublevel.batch(); + for (const download of downloads) { const formattedTitle = formatRepackName(download.title); const [firstLetter] = formattedTitle; const games = steamGames[firstLetter] || []; const gamesInSteam = games.filter((game) => - formattedTitle.startsWith(formatName(game.name)) + formattedTitle.startsWith(game.formattedName) ); if (gamesInSteam.length === 0) continue; @@ -101,9 +141,11 @@ const addNewDownloads = async ( updatedAt: now, }; - await repacksSublevel.put(`${repack.id}`, repack); + batch.put(`${repack.id}`, repack); } + await batch.write(); + const existingSource = await downloadSourcesSublevel.get( `${downloadSource.id}` ); @@ -131,6 +173,9 @@ const deleteDownloadSource = async (id: number) => { await batch.write(); await downloadSourcesSublevel.del(`${id}`); + + invalidateDownloadSourcesCache(); + invalidateIdCaches(); }; const importDownloadSource = async (url: string) => { @@ -145,11 +190,6 @@ const importDownloadSource = async (url: string) => { const now = new Date(); - const urlExistsBeforeInsert = await checkUrlExists(url); - if (urlExistsBeforeInsert) { - return; - } - const nextId = await getNextId(downloadSourcesSublevel); const downloadSource = { @@ -166,6 +206,8 @@ const importDownloadSource = async (url: string) => { await downloadSourcesSublevel.put(`${downloadSource.id}`, downloadSource); + invalidateDownloadSourcesCache(); + await addNewDownloads(downloadSource, response.data.downloads, steamGames); };