mirror of
https://github.com/hydralauncher/hydra.git
synced 2026-01-29 22:01:03 +00:00
feat: using api download sources
This commit is contained in:
Chubby Granny Chaser
@@ -1,76 +1,45 @@
|
||||
import { registerEvent } from "../register-event";
|
||||
import { downloadSourcesSublevel, repacksSublevel } from "@main/level";
|
||||
import { HydraApi, logger } from "@main/services";
|
||||
import { importDownloadSourceToLocal } from "./helpers";
|
||||
import { HydraApi } from "@main/services/hydra-api";
|
||||
import { downloadSourcesSublevel } from "@main/level";
|
||||
import type { DownloadSource } from "@types";
|
||||
|
||||
const addDownloadSource = async (
|
||||
_event: Electron.IpcMainInvokeEvent,
|
||||
url: string
|
||||
) => {
|
||||
const result = await importDownloadSourceToLocal(url, true);
|
||||
if (!result) {
|
||||
throw new Error("Failed to import download source");
|
||||
}
|
||||
|
||||
// Verify that repacks were actually written to the database (read-after-write)
|
||||
// This ensures all async operations are complete before proceeding
|
||||
let repackCount = 0;
|
||||
for await (const [, repack] of repacksSublevel.iterator()) {
|
||||
if (repack.downloadSourceId === result.id) {
|
||||
repackCount++;
|
||||
}
|
||||
}
|
||||
|
||||
await HydraApi.post("/profile/download-sources", {
|
||||
urls: [url],
|
||||
});
|
||||
|
||||
const { fingerprint } = await HydraApi.put<{ fingerprint: string }>(
|
||||
"/download-sources",
|
||||
{
|
||||
objectIds: result.objectIds,
|
||||
},
|
||||
{ needsAuth: false }
|
||||
);
|
||||
|
||||
// Update the source with fingerprint
|
||||
const updatedSource = await downloadSourcesSublevel.get(`${result.id}`);
|
||||
if (updatedSource) {
|
||||
await downloadSourcesSublevel.put(`${result.id}`, {
|
||||
...updatedSource,
|
||||
fingerprint,
|
||||
updatedAt: new Date(),
|
||||
});
|
||||
}
|
||||
|
||||
// Final verification: ensure the source with fingerprint is persisted
|
||||
const finalSource = await downloadSourcesSublevel.get(`${result.id}`);
|
||||
if (!finalSource || !finalSource.fingerprint) {
|
||||
throw new Error("Failed to persist download source with fingerprint");
|
||||
}
|
||||
|
||||
// Verify repacks still exist after fingerprint update
|
||||
let finalRepackCount = 0;
|
||||
for await (const [, repack] of repacksSublevel.iterator()) {
|
||||
if (repack.downloadSourceId === result.id) {
|
||||
finalRepackCount++;
|
||||
}
|
||||
}
|
||||
|
||||
if (finalRepackCount !== repackCount) {
|
||||
logger.warn(
|
||||
`Repack count mismatch! Before: ${repackCount}, After: ${finalRepackCount}`
|
||||
try {
|
||||
const downloadSource = await HydraApi.post<DownloadSource>(
|
||||
"/download-sources",
|
||||
{
|
||||
url,
|
||||
},
|
||||
{ needsAuth: false }
|
||||
);
|
||||
} else {
|
||||
logger.info(
|
||||
`Final verification passed: ${finalRepackCount} repacks confirmed`
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
...result,
|
||||
fingerprint,
|
||||
};
|
||||
if (HydraApi.isLoggedIn()) {
|
||||
try {
|
||||
await HydraApi.post("/profile/download-sources", {
|
||||
urls: [url],
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("Failed to add download source to profile:", error);
|
||||
}
|
||||
}
|
||||
|
||||
const downloadSourceForStorage = {
|
||||
...downloadSource,
|
||||
fingerprint: downloadSource.fingerprint || "",
|
||||
};
|
||||
await downloadSourcesSublevel.put(
|
||||
downloadSource.id,
|
||||
downloadSourceForStorage
|
||||
);
|
||||
|
||||
return downloadSource;
|
||||
} catch (error) {
|
||||
console.error("Failed to add download source:", error);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
registerEvent("addDownloadSource", addDownloadSource);
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
import { registerEvent } from "../register-event";
|
||||
import { downloadSourcesSublevel } from "@main/level";
|
||||
|
||||
const checkDownloadSourceExists = async (
|
||||
_event: Electron.IpcMainInvokeEvent,
|
||||
url: string
|
||||
): Promise<boolean> => {
|
||||
for await (const [, source] of downloadSourcesSublevel.iterator()) {
|
||||
if (source.url === url) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
};
|
||||
|
||||
registerEvent("checkDownloadSourceExists", checkDownloadSourceExists);
|
||||
@@ -1,13 +0,0 @@
|
||||
import { registerEvent } from "../register-event";
|
||||
import { downloadSourcesSublevel, repacksSublevel } from "@main/level";
|
||||
import { invalidateIdCaches } from "./helpers";
|
||||
|
||||
const deleteAllDownloadSources = async (
|
||||
_event: Electron.IpcMainInvokeEvent
|
||||
) => {
|
||||
await Promise.all([repacksSublevel.clear(), downloadSourcesSublevel.clear()]);
|
||||
|
||||
invalidateIdCaches();
|
||||
};
|
||||
|
||||
registerEvent("deleteAllDownloadSources", deleteAllDownloadSources);
|
||||
@@ -1,28 +0,0 @@
|
||||
import { registerEvent } from "../register-event";
|
||||
import { downloadSourcesSublevel, repacksSublevel } from "@main/level";
|
||||
import { invalidateIdCaches } from "./helpers";
|
||||
|
||||
const deleteDownloadSource = async (
|
||||
_event: Electron.IpcMainInvokeEvent,
|
||||
id: number
|
||||
) => {
|
||||
const repacksToDelete: string[] = [];
|
||||
|
||||
for await (const [key, repack] of repacksSublevel.iterator()) {
|
||||
if (repack.downloadSourceId === id) {
|
||||
repacksToDelete.push(key);
|
||||
}
|
||||
}
|
||||
|
||||
const batch = repacksSublevel.batch();
|
||||
for (const key of repacksToDelete) {
|
||||
batch.del(key);
|
||||
}
|
||||
await batch.write();
|
||||
|
||||
await downloadSourcesSublevel.del(`${id}`);
|
||||
|
||||
invalidateIdCaches();
|
||||
};
|
||||
|
||||
registerEvent("deleteDownloadSource", deleteDownloadSource);
|
||||
@@ -1,19 +0,0 @@
|
||||
import { registerEvent } from "../register-event";
|
||||
import { downloadSourcesSublevel, DownloadSource } from "@main/level";
|
||||
|
||||
const getDownloadSourcesList = async (_event: Electron.IpcMainInvokeEvent) => {
|
||||
const sources: DownloadSource[] = [];
|
||||
|
||||
for await (const [, source] of downloadSourcesSublevel.iterator()) {
|
||||
sources.push(source);
|
||||
}
|
||||
|
||||
// Sort by createdAt descending
|
||||
sources.sort(
|
||||
(a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime()
|
||||
);
|
||||
|
||||
return sources;
|
||||
};
|
||||
|
||||
registerEvent("getDownloadSourcesList", getDownloadSourcesList);
|
||||
@@ -1,8 +1,8 @@
|
||||
import { HydraApi } from "@main/services";
|
||||
import { downloadSourcesSublevel } from "@main/level";
|
||||
import { registerEvent } from "../register-event";
|
||||
|
||||
const getDownloadSources = async (_event: Electron.IpcMainInvokeEvent) => {
|
||||
return HydraApi.get("/profile/download-sources");
|
||||
return downloadSourcesSublevel.values().all();
|
||||
};
|
||||
|
||||
registerEvent("getDownloadSources", getDownloadSources);
|
||||
|
||||
@@ -1,314 +0,0 @@
|
||||
import axios from "axios";
|
||||
import { z } from "zod";
|
||||
import { downloadSourcesSublevel, repacksSublevel } from "@main/level";
|
||||
import { DownloadSourceStatus } from "@shared";
|
||||
import crypto from "node:crypto";
|
||||
import { logger, ResourceCache } from "@main/services";
|
||||
|
||||
export const downloadSourceSchema = z.object({
|
||||
name: z.string().max(255),
|
||||
downloads: z.array(
|
||||
z.object({
|
||||
title: z.string().max(255),
|
||||
uris: z.array(z.string()),
|
||||
uploadDate: z.string().max(255),
|
||||
fileSize: z.string().max(255),
|
||||
})
|
||||
),
|
||||
});
|
||||
|
||||
export type TitleHashMapping = Record<string, number[]>;
|
||||
|
||||
let titleHashMappingCache: TitleHashMapping | null = null;
|
||||
|
||||
export const getTitleHashMapping = async (): Promise<TitleHashMapping> => {
|
||||
if (titleHashMappingCache) {
|
||||
return titleHashMappingCache;
|
||||
}
|
||||
|
||||
try {
|
||||
const cached =
|
||||
ResourceCache.getCachedData<TitleHashMapping>("sources-manifest");
|
||||
if (cached) {
|
||||
titleHashMappingCache = cached;
|
||||
return cached;
|
||||
}
|
||||
|
||||
const fetched = await ResourceCache.fetchAndCache<TitleHashMapping>(
|
||||
"sources-manifest",
|
||||
"https://cdn.losbroxas.org/sources-manifest.json",
|
||||
10000
|
||||
);
|
||||
titleHashMappingCache = fetched;
|
||||
return fetched;
|
||||
} catch (error) {
|
||||
logger.error("Failed to fetch title hash mapping:", error);
|
||||
return {} as TitleHashMapping;
|
||||
}
|
||||
};
|
||||
|
||||
export const hashTitle = (title: string): string => {
|
||||
return crypto.createHash("sha256").update(title).digest("hex");
|
||||
};
|
||||
|
||||
export type SteamGamesByLetter = Record<string, { id: string; name: string }[]>;
|
||||
export type FormattedSteamGame = {
|
||||
id: string;
|
||||
name: string;
|
||||
formattedName: string;
|
||||
};
|
||||
export type FormattedSteamGamesByLetter = Record<string, FormattedSteamGame[]>;
|
||||
|
||||
export const formatName = (name: string) => {
|
||||
return name
|
||||
.normalize("NFD")
|
||||
.replaceAll(/[\u0300-\u036f]/g, "")
|
||||
.toLowerCase()
|
||||
.replaceAll(/[^a-z0-9]/g, "");
|
||||
};
|
||||
|
||||
export const formatRepackName = (name: string) => {
|
||||
return formatName(name.replace("[DL]", ""));
|
||||
};
|
||||
|
||||
interface DownloadSource {
|
||||
id: number;
|
||||
url: string;
|
||||
name: string;
|
||||
etag: string | null;
|
||||
status: number;
|
||||
downloadCount: number;
|
||||
objectIds: string[];
|
||||
fingerprint?: string;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
}
|
||||
|
||||
const getDownloadSourcesMap = async (): Promise<
|
||||
Map<string, DownloadSource>
|
||||
> => {
|
||||
const map = new Map();
|
||||
for await (const [key, source] of downloadSourcesSublevel.iterator()) {
|
||||
map.set(key, source);
|
||||
}
|
||||
|
||||
return map;
|
||||
};
|
||||
|
||||
export const checkUrlExists = async (url: string): Promise<boolean> => {
|
||||
const sources = await getDownloadSourcesMap();
|
||||
for (const source of sources.values()) {
|
||||
if (source.url === url) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
let steamGamesFormattedCache: FormattedSteamGamesByLetter | null = null;
|
||||
|
||||
export const getSteamGames = async (): Promise<FormattedSteamGamesByLetter> => {
|
||||
if (steamGamesFormattedCache) {
|
||||
return steamGamesFormattedCache;
|
||||
}
|
||||
|
||||
let steamGames: SteamGamesByLetter;
|
||||
|
||||
const cached = ResourceCache.getCachedData<SteamGamesByLetter>(
|
||||
"steam-games-by-letter"
|
||||
);
|
||||
if (cached) {
|
||||
steamGames = cached;
|
||||
} else {
|
||||
steamGames = await ResourceCache.fetchAndCache<SteamGamesByLetter>(
|
||||
"steam-games-by-letter",
|
||||
`${import.meta.env.MAIN_VITE_EXTERNAL_RESOURCES_URL}/steam-games-by-letter.json`
|
||||
);
|
||||
}
|
||||
|
||||
const formattedData: FormattedSteamGamesByLetter = {};
|
||||
for (const [letter, games] of Object.entries(steamGames)) {
|
||||
formattedData[letter] = games.map((game) => ({
|
||||
...game,
|
||||
formattedName: formatName(game.name),
|
||||
}));
|
||||
}
|
||||
|
||||
steamGamesFormattedCache = formattedData;
|
||||
return formattedData;
|
||||
};
|
||||
|
||||
export type SublevelIterator = AsyncIterable<[string, { id: number }]>;
|
||||
|
||||
export interface SublevelWithId {
|
||||
iterator: () => SublevelIterator;
|
||||
}
|
||||
|
||||
let maxRepackId: number | null = null;
|
||||
let maxDownloadSourceId: number | null = null;
|
||||
|
||||
export const getNextId = async (sublevel: SublevelWithId): Promise<number> => {
|
||||
const isRepackSublevel = sublevel === repacksSublevel;
|
||||
const isDownloadSourceSublevel = sublevel === downloadSourcesSublevel;
|
||||
|
||||
if (isRepackSublevel && maxRepackId !== null) {
|
||||
return ++maxRepackId;
|
||||
}
|
||||
|
||||
if (isDownloadSourceSublevel && maxDownloadSourceId !== null) {
|
||||
return ++maxDownloadSourceId;
|
||||
}
|
||||
|
||||
let maxId = 0;
|
||||
for await (const [, value] of sublevel.iterator()) {
|
||||
if (value.id > maxId) {
|
||||
maxId = value.id;
|
||||
}
|
||||
}
|
||||
|
||||
if (isRepackSublevel) {
|
||||
maxRepackId = maxId;
|
||||
} else if (isDownloadSourceSublevel) {
|
||||
maxDownloadSourceId = maxId;
|
||||
}
|
||||
|
||||
return maxId + 1;
|
||||
};
|
||||
|
||||
export const invalidateIdCaches = () => {
|
||||
maxRepackId = null;
|
||||
maxDownloadSourceId = null;
|
||||
};
|
||||
|
||||
export const addNewDownloads = async (
|
||||
downloadSource: { id: number; name: string },
|
||||
downloads: z.infer<typeof downloadSourceSchema>["downloads"],
|
||||
steamGames: FormattedSteamGamesByLetter
|
||||
) => {
|
||||
const now = new Date();
|
||||
const objectIdsOnSource = new Set<string>();
|
||||
|
||||
let nextRepackId = await getNextId(repacksSublevel);
|
||||
|
||||
const batch = repacksSublevel.batch();
|
||||
|
||||
// Get title hash mapping and perform matching in worker thread
|
||||
const titleHashMapping = await getTitleHashMapping();
|
||||
|
||||
const { GameMatcherWorkerManager } = await import("@main/services");
|
||||
const matchResult = await GameMatcherWorkerManager.matchDownloads(
|
||||
downloads,
|
||||
steamGames,
|
||||
titleHashMapping
|
||||
);
|
||||
|
||||
// Process matched results and write to database
|
||||
for (const matchedDownload of matchResult.matchedDownloads) {
|
||||
for (const id of matchedDownload.objectIds) {
|
||||
objectIdsOnSource.add(id);
|
||||
}
|
||||
|
||||
const repack = {
|
||||
id: nextRepackId++,
|
||||
objectIds: matchedDownload.objectIds,
|
||||
title: matchedDownload.title,
|
||||
uris: matchedDownload.uris,
|
||||
fileSize: matchedDownload.fileSize,
|
||||
repacker: downloadSource.name,
|
||||
uploadDate: matchedDownload.uploadDate,
|
||||
downloadSourceId: downloadSource.id,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
};
|
||||
|
||||
batch.put(`${repack.id}`, repack);
|
||||
}
|
||||
|
||||
await batch.write();
|
||||
|
||||
logger.info(
|
||||
`Matching stats for ${downloadSource.name}: Hash=${matchResult.stats.hashMatchCount}, Fuzzy=${matchResult.stats.fuzzyMatchCount}, None=${matchResult.stats.noMatchCount}`
|
||||
);
|
||||
|
||||
const existingSource = await downloadSourcesSublevel.get(
|
||||
`${downloadSource.id}`
|
||||
);
|
||||
if (existingSource) {
|
||||
await downloadSourcesSublevel.put(`${downloadSource.id}`, {
|
||||
...existingSource,
|
||||
objectIds: Array.from(objectIdsOnSource),
|
||||
});
|
||||
}
|
||||
|
||||
return Array.from(objectIdsOnSource);
|
||||
};
|
||||
|
||||
export const importDownloadSourceToLocal = async (
|
||||
url: string,
|
||||
throwOnDuplicate = false
|
||||
) => {
|
||||
const urlExists = await checkUrlExists(url);
|
||||
if (urlExists) {
|
||||
if (throwOnDuplicate) {
|
||||
throw new Error("Download source with this URL already exists");
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
const response = await axios.get<z.infer<typeof downloadSourceSchema>>(url);
|
||||
|
||||
const steamGames = await getSteamGames();
|
||||
|
||||
const now = new Date();
|
||||
|
||||
const nextId = await getNextId(downloadSourcesSublevel);
|
||||
|
||||
const downloadSource = {
|
||||
id: nextId,
|
||||
url,
|
||||
name: response.data.name,
|
||||
etag: response.headers["etag"] || null,
|
||||
status: DownloadSourceStatus.UpToDate,
|
||||
downloadCount: response.data.downloads.length,
|
||||
objectIds: [],
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
};
|
||||
|
||||
await downloadSourcesSublevel.put(`${downloadSource.id}`, downloadSource);
|
||||
|
||||
const objectIds = await addNewDownloads(
|
||||
downloadSource,
|
||||
response.data.downloads,
|
||||
steamGames
|
||||
);
|
||||
|
||||
// Invalidate ID caches after creating new repacks to prevent ID collisions
|
||||
invalidateIdCaches();
|
||||
|
||||
return {
|
||||
...downloadSource,
|
||||
objectIds,
|
||||
};
|
||||
};
|
||||
|
||||
export const updateDownloadSourcePreservingTimestamp = async (
|
||||
existingSource: DownloadSource,
|
||||
url: string
|
||||
) => {
|
||||
const response = await axios.get<z.infer<typeof downloadSourceSchema>>(url);
|
||||
|
||||
const updatedSource = {
|
||||
...existingSource,
|
||||
name: response.data.name,
|
||||
etag: response.headers["etag"] || null,
|
||||
status: DownloadSourceStatus.UpToDate,
|
||||
downloadCount: response.data.downloads.length,
|
||||
updatedAt: new Date(),
|
||||
// Preserve the original createdAt timestamp
|
||||
};
|
||||
|
||||
await downloadSourcesSublevel.put(`${existingSource.id}`, updatedSource);
|
||||
|
||||
return updatedSource;
|
||||
};
|
||||
@@ -1,18 +1,27 @@
|
||||
import { HydraApi } from "@main/services";
|
||||
import { downloadSourcesSublevel } from "@main/level";
|
||||
import { registerEvent } from "../register-event";
|
||||
|
||||
const removeDownloadSource = async (
|
||||
_event: Electron.IpcMainInvokeEvent,
|
||||
url?: string,
|
||||
removeAll = false
|
||||
removeAll = false,
|
||||
downloadSourceId?: string
|
||||
) => {
|
||||
const params = new URLSearchParams({
|
||||
all: removeAll.toString(),
|
||||
});
|
||||
|
||||
if (url) params.set("url", url);
|
||||
if (downloadSourceId) params.set("downloadSourceId", downloadSourceId);
|
||||
|
||||
return HydraApi.delete(`/profile/download-sources?${params.toString()}`);
|
||||
if (HydraApi.isLoggedIn()) {
|
||||
void HydraApi.delete(`/profile/download-sources?${params.toString()}`);
|
||||
}
|
||||
|
||||
if (removeAll) {
|
||||
await downloadSourcesSublevel.clear();
|
||||
} else if (downloadSourceId) {
|
||||
await downloadSourcesSublevel.del(downloadSourceId);
|
||||
}
|
||||
};
|
||||
|
||||
registerEvent("removeDownloadSource", removeDownloadSource);
|
||||
|
||||
@@ -1,19 +0,0 @@
|
||||
import { HydraApi, logger } from "@main/services";
|
||||
import { importDownloadSourceToLocal, checkUrlExists } from "./helpers";
|
||||
|
||||
export const syncDownloadSourcesFromApi = async () => {
|
||||
try {
|
||||
const apiSources = await HydraApi.get<
|
||||
{ url: string; createdAt: string; updatedAt: string }[]
|
||||
>("/profile/download-sources");
|
||||
|
||||
for (const apiSource of apiSources) {
|
||||
const exists = await checkUrlExists(apiSource.url);
|
||||
if (!exists) {
|
||||
await importDownloadSourceToLocal(apiSource.url, false);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error("Failed to sync download sources from API:", error);
|
||||
}
|
||||
};
|
||||
@@ -1,105 +1,24 @@
|
||||
import { HydraApi } from "@main/services";
|
||||
import { registerEvent } from "../register-event";
|
||||
import axios, { AxiosError } from "axios";
|
||||
import { downloadSourcesSublevel, repacksSublevel } from "@main/level";
|
||||
import { DownloadSourceStatus } from "@shared";
|
||||
import {
|
||||
invalidateIdCaches,
|
||||
downloadSourceSchema,
|
||||
getSteamGames,
|
||||
addNewDownloads,
|
||||
} from "./helpers";
|
||||
import { downloadSourcesSublevel } from "@main/level";
|
||||
import type { DownloadSource } from "@types";
|
||||
|
||||
const syncDownloadSources = async (
|
||||
_event: Electron.IpcMainInvokeEvent
|
||||
): Promise<number> => {
|
||||
let newRepacksCount = 0;
|
||||
const syncDownloadSources = async (_event: Electron.IpcMainInvokeEvent) => {
|
||||
const downloadSources = await downloadSourcesSublevel.values().all();
|
||||
|
||||
try {
|
||||
const downloadSources: Array<{
|
||||
id: number;
|
||||
url: string;
|
||||
name: string;
|
||||
etag: string | null;
|
||||
status: number;
|
||||
downloadCount: number;
|
||||
objectIds: string[];
|
||||
fingerprint?: string;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
}> = [];
|
||||
for await (const [, source] of downloadSourcesSublevel.iterator()) {
|
||||
downloadSources.push(source);
|
||||
}
|
||||
const response = await HydraApi.post<DownloadSource[]>(
|
||||
"/download-sources/sync",
|
||||
{
|
||||
ids: downloadSources.map((downloadSource) => downloadSource.id),
|
||||
},
|
||||
{ needsAuth: false }
|
||||
);
|
||||
|
||||
// Use a Set for O(1) lookups instead of O(n) with array.some()
|
||||
const existingRepackTitles = new Set<string>();
|
||||
for await (const [, repack] of repacksSublevel.iterator()) {
|
||||
existingRepackTitles.add(repack.title);
|
||||
}
|
||||
|
||||
// Handle sources with missing fingerprints individually, don't delete all sources
|
||||
const sourcesWithFingerprints = downloadSources.filter(
|
||||
(source) => source.fingerprint
|
||||
);
|
||||
const sourcesWithoutFingerprints = downloadSources.filter(
|
||||
(source) => !source.fingerprint
|
||||
);
|
||||
|
||||
// For sources without fingerprints, just continue with normal sync
|
||||
// They will get fingerprints updated later by updateMissingFingerprints
|
||||
const allSourcesToSync = [
|
||||
...sourcesWithFingerprints,
|
||||
...sourcesWithoutFingerprints,
|
||||
];
|
||||
|
||||
for (const downloadSource of allSourcesToSync) {
|
||||
const headers: Record<string, string> = {};
|
||||
|
||||
if (downloadSource.etag) {
|
||||
headers["If-None-Match"] = downloadSource.etag;
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await axios.get(downloadSource.url, {
|
||||
headers,
|
||||
});
|
||||
|
||||
const source = downloadSourceSchema.parse(response.data);
|
||||
const steamGames = await getSteamGames();
|
||||
|
||||
// O(1) lookup instead of O(n) - massive performance improvement
|
||||
const repacks = source.downloads.filter(
|
||||
(download) => !existingRepackTitles.has(download.title)
|
||||
);
|
||||
|
||||
await downloadSourcesSublevel.put(`${downloadSource.id}`, {
|
||||
...downloadSource,
|
||||
etag: response.headers["etag"] || null,
|
||||
downloadCount: source.downloads.length,
|
||||
status: DownloadSourceStatus.UpToDate,
|
||||
});
|
||||
|
||||
await addNewDownloads(downloadSource, repacks, steamGames);
|
||||
|
||||
newRepacksCount += repacks.length;
|
||||
} catch (err: unknown) {
|
||||
const isNotModified = (err as AxiosError).response?.status === 304;
|
||||
|
||||
await downloadSourcesSublevel.put(`${downloadSource.id}`, {
|
||||
...downloadSource,
|
||||
status: isNotModified
|
||||
? DownloadSourceStatus.UpToDate
|
||||
: DownloadSourceStatus.Errored,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
invalidateIdCaches();
|
||||
|
||||
return newRepacksCount;
|
||||
} catch (err) {
|
||||
return -1;
|
||||
for (const downloadSource of response) {
|
||||
await downloadSourcesSublevel.put(downloadSource.id, downloadSource);
|
||||
}
|
||||
|
||||
return response;
|
||||
};
|
||||
|
||||
registerEvent("syncDownloadSources", syncDownloadSources);
|
||||
|
||||
@@ -1,67 +0,0 @@
|
||||
import { registerEvent } from "../register-event";
|
||||
import { downloadSourcesSublevel } from "@main/level";
|
||||
import { HydraApi, logger } from "@main/services";
|
||||
|
||||
const updateMissingFingerprints = async (
|
||||
_event: Electron.IpcMainInvokeEvent
|
||||
): Promise<number> => {
|
||||
const sourcesNeedingFingerprints: Array<{
|
||||
id: number;
|
||||
objectIds: string[];
|
||||
}> = [];
|
||||
|
||||
for await (const [, source] of downloadSourcesSublevel.iterator()) {
|
||||
if (
|
||||
!source.fingerprint &&
|
||||
source.objectIds &&
|
||||
source.objectIds.length > 0
|
||||
) {
|
||||
sourcesNeedingFingerprints.push({
|
||||
id: source.id,
|
||||
objectIds: source.objectIds,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (sourcesNeedingFingerprints.length === 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
logger.info(
|
||||
`Updating fingerprints for ${sourcesNeedingFingerprints.length} sources`
|
||||
);
|
||||
|
||||
await Promise.all(
|
||||
sourcesNeedingFingerprints.map(async (source) => {
|
||||
try {
|
||||
const { fingerprint } = await HydraApi.put<{ fingerprint: string }>(
|
||||
"/download-sources",
|
||||
{
|
||||
objectIds: source.objectIds,
|
||||
},
|
||||
{ needsAuth: false }
|
||||
);
|
||||
|
||||
const existingSource = await downloadSourcesSublevel.get(
|
||||
`${source.id}`
|
||||
);
|
||||
if (existingSource) {
|
||||
await downloadSourcesSublevel.put(`${source.id}`, {
|
||||
...existingSource,
|
||||
fingerprint,
|
||||
updatedAt: new Date(),
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
`Failed to update fingerprint for source ${source.id}:`,
|
||||
error
|
||||
);
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
return sourcesNeedingFingerprints.length;
|
||||
};
|
||||
|
||||
registerEvent("updateMissingFingerprints", updateMissingFingerprints);
|
||||
@@ -1,32 +0,0 @@
|
||||
import { registerEvent } from "../register-event";
|
||||
import axios from "axios";
|
||||
import { z } from "zod";
|
||||
|
||||
const downloadSourceSchema = z.object({
|
||||
name: z.string().max(255),
|
||||
downloads: z.array(
|
||||
z.object({
|
||||
title: z.string().max(255),
|
||||
uris: z.array(z.string()),
|
||||
uploadDate: z.string().max(255),
|
||||
fileSize: z.string().max(255),
|
||||
})
|
||||
),
|
||||
});
|
||||
|
||||
const validateDownloadSource = async (
|
||||
_event: Electron.IpcMainInvokeEvent,
|
||||
url: string
|
||||
) => {
|
||||
const response = await axios.get<z.infer<typeof downloadSourceSchema>>(url);
|
||||
|
||||
const { name } = downloadSourceSchema.parse(response.data);
|
||||
|
||||
return {
|
||||
name,
|
||||
etag: response.headers["etag"] || null,
|
||||
downloadCount: response.data.downloads.length,
|
||||
};
|
||||
};
|
||||
|
||||
registerEvent("validateDownloadSource", validateDownloadSource);
|
||||
@@ -63,14 +63,7 @@ import "./autoupdater/restart-and-install-update";
|
||||
import "./user-preferences/authenticate-real-debrid";
|
||||
import "./user-preferences/authenticate-torbox";
|
||||
import "./download-sources/add-download-source";
|
||||
import "./download-sources/update-missing-fingerprints";
|
||||
import "./download-sources/delete-download-source";
|
||||
import "./download-sources/delete-all-download-sources";
|
||||
import "./download-sources/validate-download-source";
|
||||
import "./download-sources/sync-download-sources";
|
||||
import "./download-sources/get-download-sources-list";
|
||||
import "./download-sources/check-download-source-exists";
|
||||
import "./repacks/get-all-repacks";
|
||||
import "./auth/sign-out";
|
||||
import "./auth/open-auth-window";
|
||||
import "./auth/get-session-hash";
|
||||
|
||||
@@ -1,16 +0,0 @@
|
||||
import { registerEvent } from "../register-event";
|
||||
import { repacksSublevel, GameRepack } from "@main/level";
|
||||
|
||||
const getAllRepacks = async (_event: Electron.IpcMainInvokeEvent) => {
|
||||
const repacks: GameRepack[] = [];
|
||||
|
||||
for await (const [, repack] of repacksSublevel.iterator()) {
|
||||
if (Array.isArray(repack.objectIds)) {
|
||||
repacks.push(repack);
|
||||
}
|
||||
}
|
||||
|
||||
return repacks;
|
||||
};
|
||||
|
||||
registerEvent("getAllRepacks", getAllRepacks);
|
||||
Reference in New Issue
Block a user