feat: improving caching

This commit is contained in:
Chubby Granny Chaser
2025-10-15 13:58:40 +01:00
parent 136a44473f
commit 24106eaeab
35 changed files with 246 additions and 1061 deletions

View File

@@ -1,10 +1,7 @@
import { registerEvent } from "../register-event";
import { downloadSourcesSublevel, repacksSublevel } from "@main/level";
import { HydraApi } from "@main/services";
import {
importDownloadSourceToLocal,
invalidateDownloadSourcesCache,
} from "./helpers";
import { HydraApi, logger } from "@main/services";
import { importDownloadSourceToLocal } from "./helpers";
const addDownloadSource = async (
_event: Electron.IpcMainInvokeEvent,
@@ -26,17 +23,6 @@ const addDownloadSource = async (
}
}
// Log for debugging - helps identify if repacks are being created
console.log(
`✅ Download source ${result.id} (${result.name}) created with ${repackCount} repacks`
);
console.log(
` Repack IDs: [${repackIds.slice(0, 5).join(", ")}${repackIds.length > 5 ? "..." : ""}]`
);
console.log(
` Object IDs: [${result.objectIds.slice(0, 5).join(", ")}${result.objectIds.length > 5 ? "..." : ""}]`
);
await HydraApi.post("/profile/download-sources", {
urls: [url],
});
@@ -74,18 +60,15 @@ const addDownloadSource = async (
}
if (finalRepackCount !== repackCount) {
console.warn(
`⚠️ Repack count mismatch! Before: ${repackCount}, After: ${finalRepackCount}`
logger.warn(
`Repack count mismatch! Before: ${repackCount}, After: ${finalRepackCount}`
);
} else {
console.log(
`Final verification passed: ${finalRepackCount} repacks confirmed`
logger.info(
`Final verification passed: ${finalRepackCount} repacks confirmed`
);
}
// Invalidate cache to ensure fresh data on next read
invalidateDownloadSourcesCache();
return {
...result,
fingerprint,

View File

@@ -1,14 +1,12 @@
import { registerEvent } from "../register-event";
import { downloadSourcesSublevel, repacksSublevel } from "@main/level";
import { invalidateDownloadSourcesCache, invalidateIdCaches } from "./helpers";
import { invalidateIdCaches } from "./helpers";
const deleteAllDownloadSources = async (
_event: Electron.IpcMainInvokeEvent
) => {
await Promise.all([repacksSublevel.clear(), downloadSourcesSublevel.clear()]);
// Invalidate caches after clearing all sources
invalidateDownloadSourcesCache();
invalidateIdCaches();
};

View File

@@ -1,6 +1,6 @@
import { registerEvent } from "../register-event";
import { downloadSourcesSublevel, repacksSublevel } from "@main/level";
import { invalidateDownloadSourcesCache, invalidateIdCaches } from "./helpers";
import { invalidateIdCaches } from "./helpers";
const deleteDownloadSource = async (
_event: Electron.IpcMainInvokeEvent,
@@ -22,8 +22,6 @@ const deleteDownloadSource = async (
await downloadSourcesSublevel.del(`${id}`);
// Invalidate caches after deletion
invalidateDownloadSourcesCache();
invalidateIdCaches();
};

View File

@@ -3,8 +3,9 @@ import { z } from "zod";
import { downloadSourcesSublevel, repacksSublevel } from "@main/level";
import { DownloadSourceStatus } from "@shared";
import crypto from "crypto";
import { logger, ResourceCache } from "@main/services";
const downloadSourceSchema = z.object({
export const downloadSourceSchema = z.object({
name: z.string().max(255),
downloads: z.array(
z.object({
@@ -16,51 +17,49 @@ const downloadSourceSchema = z.object({
),
});
// Pre-computed title-to-Steam-ID mapping
type TitleHashMapping = Record<string, number[]>;
let titleHashMappingCache: TitleHashMapping | null = null;
let titleHashMappingCacheTime = 0;
const TITLE_HASH_MAPPING_TTL = 86400000; // 24 hours
export type TitleHashMapping = Record<string, number[]>;
const getTitleHashMapping = async (): Promise<TitleHashMapping> => {
const now = Date.now();
if (
titleHashMappingCache &&
now - titleHashMappingCacheTime < TITLE_HASH_MAPPING_TTL
) {
let titleHashMappingCache: TitleHashMapping | null = null;
export const getTitleHashMapping = async (): Promise<TitleHashMapping> => {
if (titleHashMappingCache) {
return titleHashMappingCache;
}
try {
const response = await axios.get<TitleHashMapping>(
"https://cdn.losbroxas.org/results_a4c50f70c2.json",
{
timeout: 10000,
}
);
const cached =
ResourceCache.getCachedData<TitleHashMapping>("sources-manifest");
if (cached) {
titleHashMappingCache = cached;
return cached;
}
titleHashMappingCache = response.data;
titleHashMappingCacheTime = now;
console.log(
`✅ Loaded title hash mapping with ${Object.keys(response.data).length} entries`
const fetched = await ResourceCache.fetchAndCache<TitleHashMapping>(
"sources-manifest",
"https://cdn.losbroxas.org/sources-manifest.json",
10000
);
return response.data;
titleHashMappingCache = fetched;
return fetched;
} catch (error) {
console.error("Failed to fetch title hash mapping:", error);
// Return empty mapping on error - will fall back to fuzzy matching
return {};
logger.error("Failed to fetch title hash mapping:", error);
return {} as TitleHashMapping;
}
};
const hashTitle = (title: string): string => {
export const hashTitle = (title: string): string => {
return crypto.createHash("sha256").update(title).digest("hex");
};
type SteamGamesByLetter = Record<string, { id: string; name: string }[]>;
type FormattedSteamGame = { id: string; name: string; formattedName: string };
type FormattedSteamGamesByLetter = Record<string, FormattedSteamGame[]>;
export type SteamGamesByLetter = Record<string, { id: string; name: string }[]>;
export type FormattedSteamGame = {
id: string;
name: string;
formattedName: string;
};
export type FormattedSteamGamesByLetter = Record<string, FormattedSteamGame[]>;
const formatName = (name: string) => {
export const formatName = (name: string) => {
return name
.normalize("NFD")
.replace(/[\u0300-\u036f]/g, "")
@@ -68,7 +67,7 @@ const formatName = (name: string) => {
.replace(/[^a-z0-9]/g, "");
};
const formatRepackName = (name: string) => {
export const formatRepackName = (name: string) => {
return formatName(name.replace("[DL]", ""));
};
@@ -85,32 +84,17 @@ interface DownloadSource {
updatedAt: Date;
}
let downloadSourcesCache: Map<string, DownloadSource> | null = null;
let downloadSourcesCacheTime = 0;
const CACHE_TTL = 5000;
const getDownloadSourcesMap = async (): Promise<
Map<string, DownloadSource>
> => {
const now = Date.now();
if (downloadSourcesCache && now - downloadSourcesCacheTime < CACHE_TTL) {
return downloadSourcesCache;
}
const map = new Map();
for await (const [key, source] of downloadSourcesSublevel.iterator()) {
map.set(key, source);
}
downloadSourcesCache = map;
downloadSourcesCacheTime = now;
return map;
};
export const invalidateDownloadSourcesCache = () => {
downloadSourcesCache = null;
};
export const checkUrlExists = async (url: string): Promise<boolean> => {
const sources = await getDownloadSourcesMap();
for (const source of sources.values()) {
@@ -121,43 +105,49 @@ export const checkUrlExists = async (url: string): Promise<boolean> => {
return false;
};
let steamGamesCache: FormattedSteamGamesByLetter | null = null;
let steamGamesCacheTime = 0;
const STEAM_GAMES_CACHE_TTL = 300000;
let steamGamesFormattedCache: FormattedSteamGamesByLetter | null = null;
const getSteamGames = async (): Promise<FormattedSteamGamesByLetter> => {
const now = Date.now();
if (steamGamesCache && now - steamGamesCacheTime < STEAM_GAMES_CACHE_TTL) {
return steamGamesCache;
export const getSteamGames = async (): Promise<FormattedSteamGamesByLetter> => {
if (steamGamesFormattedCache) {
return steamGamesFormattedCache;
}
const response = await axios.get<SteamGamesByLetter>(
`${import.meta.env.MAIN_VITE_EXTERNAL_RESOURCES_URL}/steam-games-by-letter.json`
let steamGames: SteamGamesByLetter;
const cached = ResourceCache.getCachedData<SteamGamesByLetter>(
"steam-games-by-letter"
);
if (cached) {
steamGames = cached;
} else {
steamGames = await ResourceCache.fetchAndCache<SteamGamesByLetter>(
"steam-games-by-letter",
`${import.meta.env.MAIN_VITE_EXTERNAL_RESOURCES_URL}/steam-games-by-letter.json`
);
}
const formattedData: FormattedSteamGamesByLetter = {};
for (const [letter, games] of Object.entries(response.data)) {
for (const [letter, games] of Object.entries(steamGames)) {
formattedData[letter] = games.map((game) => ({
...game,
formattedName: formatName(game.name),
}));
}
steamGamesCache = formattedData;
steamGamesCacheTime = now;
steamGamesFormattedCache = formattedData;
return formattedData;
};
type SublevelIterator = AsyncIterable<[string, { id: number }]>;
export type SublevelIterator = AsyncIterable<[string, { id: number }]>;
interface SublevelWithId {
export interface SublevelWithId {
iterator: () => SublevelIterator;
}
let maxRepackId: number | null = null;
let maxDownloadSourceId: number | null = null;
const getNextId = async (sublevel: SublevelWithId): Promise<number> => {
export const getNextId = async (sublevel: SublevelWithId): Promise<number> => {
const isRepackSublevel = sublevel === repacksSublevel;
const isDownloadSourceSublevel = sublevel === downloadSourcesSublevel;
@@ -190,7 +180,7 @@ export const invalidateIdCaches = () => {
maxDownloadSourceId = null;
};
const addNewDownloads = async (
export const addNewDownloads = async (
downloadSource: { id: number; name: string },
downloads: z.infer<typeof downloadSourceSchema>["downloads"],
steamGames: FormattedSteamGamesByLetter
@@ -202,7 +192,6 @@ const addNewDownloads = async (
const batch = repacksSublevel.batch();
// Fetch the pre-computed hash mapping
const titleHashMapping = await getTitleHashMapping();
let hashMatchCount = 0;
let fuzzyMatchCount = 0;
@@ -212,20 +201,16 @@ const addNewDownloads = async (
let objectIds: string[] = [];
let usedHashMatch = false;
// FIRST: Try hash-based lookup (fast and accurate)
const titleHash = hashTitle(download.title);
const steamIdsFromHash = titleHashMapping[titleHash];
if (steamIdsFromHash && steamIdsFromHash.length > 0) {
// Found in hash mapping - trust it completely
hashMatchCount++;
usedHashMatch = true;
// Use the Steam IDs directly as strings (trust the hash mapping)
objectIds = steamIdsFromHash.map(String);
}
// FALLBACK: Use fuzzy matching ONLY if hash lookup found nothing
if (!usedHashMatch) {
let gamesInSteam: FormattedSteamGame[] = [];
const formattedTitle = formatRepackName(download.title);
@@ -234,12 +219,10 @@ const addNewDownloads = async (
const [firstLetter] = formattedTitle;
const games = steamGames[firstLetter] || [];
// Try exact prefix match first
gamesInSteam = games.filter((game) =>
formattedTitle.startsWith(game.formattedName)
);
// If no exact prefix match, try contains match (more lenient)
if (gamesInSteam.length === 0) {
gamesInSteam = games.filter(
(game) =>
@@ -248,7 +231,6 @@ const addNewDownloads = async (
);
}
// If still no match, try checking all letters (not just first letter)
if (gamesInSteam.length === 0) {
for (const letter of Object.keys(steamGames)) {
const letterGames = steamGames[letter] || [];
@@ -275,13 +257,10 @@ const addNewDownloads = async (
}
}
// Add matched game IDs to source tracking
for (const id of objectIds) {
objectIdsOnSource.add(id);
}
// Create the repack even if no games matched
// This ensures all repacks from sources are imported
const repack = {
id: nextRepackId++,
objectIds: objectIds,
@@ -300,9 +279,8 @@ const addNewDownloads = async (
await batch.write();
// Log matching statistics
console.log(
`📊 Matching stats for ${downloadSource.name}: Hash=${hashMatchCount}, Fuzzy=${fuzzyMatchCount}, None=${noMatchCount}`
logger.info(
`Matching stats for ${downloadSource.name}: Hash=${hashMatchCount}, Fuzzy=${fuzzyMatchCount}, None=${noMatchCount}`
);
const existingSource = await downloadSourcesSublevel.get(
@@ -352,8 +330,6 @@ export const importDownloadSourceToLocal = async (
await downloadSourcesSublevel.put(`${downloadSource.id}`, downloadSource);
invalidateDownloadSourcesCache();
const objectIds = await addNewDownloads(
downloadSource,
response.data.downloads,
@@ -387,7 +363,5 @@ export const updateDownloadSourcePreservingTimestamp = async (
await downloadSourcesSublevel.put(`${existingSource.id}`, updatedSource);
invalidateDownloadSourcesCache();
return updatedSource;
};

View File

@@ -1,4 +1,4 @@
import { HydraApi } from "@main/services";
import { HydraApi, logger } from "@main/services";
import { importDownloadSourceToLocal, checkUrlExists } from "./helpers";
export const syncDownloadSourcesFromApi = async () => {
@@ -14,6 +14,6 @@ export const syncDownloadSourcesFromApi = async () => {
}
}
} catch (error) {
console.error("Failed to sync download sources from API:", error);
logger.error("Failed to sync download sources from API:", error);
}
};

View File

@@ -1,267 +1,13 @@
import { registerEvent } from "../register-event";
import axios, { AxiosError } from "axios";
import { z } from "zod";
import { downloadSourcesSublevel, repacksSublevel } from "@main/level";
import { DownloadSourceStatus } from "@shared";
import { invalidateDownloadSourcesCache, invalidateIdCaches } from "./helpers";
import crypto from "crypto";
const downloadSourceSchema = z.object({
name: z.string().max(255),
downloads: z.array(
z.object({
title: z.string().max(255),
uris: z.array(z.string()),
uploadDate: z.string().max(255),
fileSize: z.string().max(255),
})
),
});
// Pre-computed title-to-Steam-ID mapping (shared with helpers.ts)
type TitleHashMapping = Record<string, number[]>;
let titleHashMappingCache: TitleHashMapping | null = null;
let titleHashMappingCacheTime = 0;
const TITLE_HASH_MAPPING_TTL = 86400000; // 24 hours
const getTitleHashMapping = async (): Promise<TitleHashMapping> => {
const now = Date.now();
if (
titleHashMappingCache &&
now - titleHashMappingCacheTime < TITLE_HASH_MAPPING_TTL
) {
return titleHashMappingCache;
}
try {
const response = await axios.get<TitleHashMapping>(
"https://cdn.losbroxas.org/results_a4c50f70c2.json",
{
timeout: 10000,
}
);
titleHashMappingCache = response.data;
titleHashMappingCacheTime = now;
console.log(
`✅ Loaded title hash mapping with ${Object.keys(response.data).length} entries`
);
return response.data;
} catch (error) {
console.error("Failed to fetch title hash mapping:", error);
return {};
}
};
const hashTitle = (title: string): string => {
return crypto.createHash("sha256").update(title).digest("hex");
};
type SteamGamesByLetter = Record<string, { id: string; name: string }[]>;
type FormattedSteamGame = { id: string; name: string; formattedName: string };
type FormattedSteamGamesByLetter = Record<string, FormattedSteamGame[]>;
const formatName = (name: string) => {
return name
.normalize("NFD")
.replace(/[\u0300-\u036f]/g, "")
.toLowerCase()
.replace(/[^a-z0-9]/g, "");
};
const formatRepackName = (name: string) => {
return formatName(name.replace("[DL]", ""));
};
let steamGamesCache: FormattedSteamGamesByLetter | null = null;
let steamGamesCacheTime = 0;
const STEAM_GAMES_CACHE_TTL = 300000;
const getSteamGames = async (): Promise<FormattedSteamGamesByLetter> => {
const now = Date.now();
if (steamGamesCache && now - steamGamesCacheTime < STEAM_GAMES_CACHE_TTL) {
return steamGamesCache;
}
const response = await axios.get<SteamGamesByLetter>(
`${import.meta.env.MAIN_VITE_EXTERNAL_RESOURCES_URL}/steam-games-by-letter.json`
);
const formattedData: FormattedSteamGamesByLetter = {};
for (const [letter, games] of Object.entries(response.data)) {
formattedData[letter] = games.map((game) => ({
...game,
formattedName: formatName(game.name),
}));
}
steamGamesCache = formattedData;
steamGamesCacheTime = now;
return formattedData;
};
type SublevelIterator = AsyncIterable<[string, { id: number }]>;
interface SublevelWithId {
iterator: () => SublevelIterator;
}
let maxRepackId: number | null = null;
let maxDownloadSourceId: number | null = null;
const getNextId = async (sublevel: SublevelWithId): Promise<number> => {
const isRepackSublevel = sublevel === repacksSublevel;
const isDownloadSourceSublevel = sublevel === downloadSourcesSublevel;
if (isRepackSublevel && maxRepackId !== null) {
return ++maxRepackId;
}
if (isDownloadSourceSublevel && maxDownloadSourceId !== null) {
return ++maxDownloadSourceId;
}
let maxId = 0;
for await (const [, value] of sublevel.iterator()) {
if (value.id > maxId) {
maxId = value.id;
}
}
if (isRepackSublevel) {
maxRepackId = maxId;
} else if (isDownloadSourceSublevel) {
maxDownloadSourceId = maxId;
}
return maxId + 1;
};
const addNewDownloads = async (
downloadSource: { id: number; name: string },
downloads: z.infer<typeof downloadSourceSchema>["downloads"],
steamGames: FormattedSteamGamesByLetter
) => {
const now = new Date();
const objectIdsOnSource = new Set<string>();
let nextRepackId = await getNextId(repacksSublevel);
const batch = repacksSublevel.batch();
// Fetch the pre-computed hash mapping
const titleHashMapping = await getTitleHashMapping();
let hashMatchCount = 0;
let fuzzyMatchCount = 0;
let noMatchCount = 0;
for (const download of downloads) {
let objectIds: string[] = [];
let usedHashMatch = false;
// FIRST: Try hash-based lookup (fast and accurate)
const titleHash = hashTitle(download.title);
const steamIdsFromHash = titleHashMapping[titleHash];
if (steamIdsFromHash && steamIdsFromHash.length > 0) {
// Found in hash mapping - trust it completely
hashMatchCount++;
usedHashMatch = true;
// Use the Steam IDs directly as strings (trust the hash mapping)
objectIds = steamIdsFromHash.map(String);
}
// FALLBACK: Use fuzzy matching ONLY if hash lookup found nothing
if (!usedHashMatch) {
let gamesInSteam: FormattedSteamGame[] = [];
const formattedTitle = formatRepackName(download.title);
if (formattedTitle && formattedTitle.length > 0) {
const [firstLetter] = formattedTitle;
const games = steamGames[firstLetter] || [];
// Try exact prefix match first
gamesInSteam = games.filter((game) =>
formattedTitle.startsWith(game.formattedName)
);
// If no exact prefix match, try contains match (more lenient)
if (gamesInSteam.length === 0) {
gamesInSteam = games.filter(
(game) =>
formattedTitle.includes(game.formattedName) ||
game.formattedName.includes(formattedTitle)
);
}
// If still no match, try checking all letters (not just first letter)
if (gamesInSteam.length === 0) {
for (const letter of Object.keys(steamGames)) {
const letterGames = steamGames[letter] || [];
const matches = letterGames.filter(
(game) =>
formattedTitle.includes(game.formattedName) ||
game.formattedName.includes(formattedTitle)
);
if (matches.length > 0) {
gamesInSteam = matches;
break;
}
}
}
if (gamesInSteam.length > 0) {
fuzzyMatchCount++;
objectIds = gamesInSteam.map((game) => String(game.id));
} else {
noMatchCount++;
}
} else {
noMatchCount++;
}
}
// Add matched game IDs to source tracking
for (const id of objectIds) {
objectIdsOnSource.add(id);
}
// Create the repack even if no games matched
// This ensures all repacks from sources are imported
const repack = {
id: nextRepackId++,
objectIds: objectIds,
title: download.title,
uris: download.uris,
fileSize: download.fileSize,
repacker: downloadSource.name,
uploadDate: download.uploadDate,
downloadSourceId: downloadSource.id,
createdAt: now,
updatedAt: now,
};
batch.put(`${repack.id}`, repack);
}
await batch.write();
// Log matching statistics
console.log(
`📊 Matching stats for ${downloadSource.name}: Hash=${hashMatchCount}, Fuzzy=${fuzzyMatchCount}, None=${noMatchCount}`
);
const existingSource = await downloadSourcesSublevel.get(
`${downloadSource.id}`
);
if (existingSource) {
await downloadSourcesSublevel.put(`${downloadSource.id}`, {
...existingSource,
objectIds: Array.from(objectIdsOnSource),
});
}
};
import {
invalidateIdCaches,
downloadSourceSchema,
getSteamGames,
addNewDownloads,
} from "./helpers";
const syncDownloadSources = async (
_event: Electron.IpcMainInvokeEvent
@@ -358,8 +104,6 @@ const syncDownloadSources = async (
}
}
// Invalidate caches after all sync operations complete
invalidateDownloadSourcesCache();
invalidateIdCaches();
return newRepacksCount;

View File

@@ -1,6 +1,6 @@
import { registerEvent } from "../register-event";
import { downloadSourcesSublevel } from "@main/level";
import { HydraApi } from "@main/services";
import { HydraApi, logger } from "@main/services";
const updateMissingFingerprints = async (
_event: Electron.IpcMainInvokeEvent
@@ -27,7 +27,7 @@ const updateMissingFingerprints = async (
return 0;
}
console.log(
logger.info(
`Updating fingerprints for ${sourcesNeedingFingerprints.length} sources`
);
@@ -53,7 +53,7 @@ const updateMissingFingerprints = async (
});
}
} catch (error) {
console.error(
logger.error(
`Failed to update fingerprint for source ${source.id}:`,
error
);

View File

@@ -61,7 +61,6 @@ import "./user-preferences/auto-launch";
import "./autoupdater/check-for-updates";
import "./autoupdater/restart-and-install-update";
import "./user-preferences/authenticate-real-debrid";
import "./user-preferences/authenticate-all-debrid";
import "./user-preferences/authenticate-torbox";
import "./download-sources/add-download-source";
import "./download-sources/update-missing-fingerprints";

View File

@@ -1,17 +0,0 @@
import { AllDebridClient } from "@main/services/download/all-debrid";
import { registerEvent } from "../register-event";
const authenticateAllDebrid = async (
_event: Electron.IpcMainInvokeEvent,
apiKey: string
) => {
AllDebridClient.authorize(apiKey);
const result = await AllDebridClient.getUser();
if ("error_code" in result) {
return { error_code: result.error_code };
}
return result.user;
};
registerEvent("authenticateAllDebrid", authenticateAllDebrid);