Lightemerald/hydra
1
0
Fork 0
mirror of https://github.com/hydralauncher/hydra.git synced 2026-01-22 18:33:56 +00:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

feat: adding import download source

Browse Source
This commit is contained in:
Chubby Granny Chaser
2024-06-03 02:12:05 +01:00
parent ddd9ea69df
commit 48e07370e4
70 changed files with 925 additions and 1261 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/main/services/download-manager.ts
View File

@@ -192,7 +192,6 @@ export class DownloadManager {
const game = await gameRepository.findOne({
where: { id: this.game.id, isDeleted: false },
relations: { repack: true },
});
if (progress === 1 && this.game && !isDownloadingMetadata) {
@@ -291,10 +290,10 @@ export class DownloadManager {
if (game.downloader === Downloader.RealDebrid) {
this.realDebridTorrentId = await RealDebridClient.getTorrentId(
game!.repack.magnet
game!.uri!
);
} else {
this.gid = await this.aria2.call("addUri", [game.repack.magnet], options);
this.gid = await this.aria2.call("addUri", [game.uri!], options);
this.downloads.set(game.id, this.gid);
}

4
src/main/services/how-long-to-beat.ts
View File

@@ -1,8 +1,8 @@
import { formatName } from "@main/helpers";
import axios from "axios";
import { JSDOM } from "jsdom";
import { requestWebPage } from "./repack-tracker/helpers";
import { requestWebPage } from "@main/helpers";
import { HowLongToBeatCategory } from "@types";
import { formatName } from "@shared";
export interface HowLongToBeatResult {
game_id: number;

3
src/main/services/index.ts
View File

@@ -1,11 +1,10 @@
export * from "./logger";
export * from "./repack-tracker";
export * from "./steam";
export * from "./steam-250";
export * from "./steam-grid";
export * from "./update-resolver";
export * from "./window-manager";
export * from "./download-manager";
export * from "./how-long-to-beat";
export * from "./process-watcher";
export * from "./main-loop";
export * from "./search-engine";

146
src/main/services/repack-tracker/1337x.ts
View File

@@ -1,146 +0,0 @@
import { JSDOM } from "jsdom";
import { Repack } from "@main/entity";
import { requestWebPage, savePage } from "./helpers";
const months = [
"Jan",
"Feb",
"Mar",
"Apr",
"May",
"Jun",
"Jul",
"Aug",
"Sep",
"Oct",
"Nov",
"Dec",
];
export const request1337x = async (path: string) =>
requestWebPage(`https://1337xx.to${path}`);
const formatUploadDate = (str: string) => {
const date = new Date();
const [month, day, year] = str.split(" ");
date.setMonth(months.indexOf(month.replace(".", "")));
date.setDate(Number(day.substring(0, 2)));
date.setFullYear(Number("20" + year.replace("'", "")));
date.setHours(0, 0, 0, 0);
return date;
};
/* TODO: $a will often be null */
const getTorrentDetails = async (path: string) => {
const response = await request1337x(path);
const { window } = new JSDOM(response);
const { document } = window;
const $a = window.document.querySelector(
".torrentdown1"
) as HTMLAnchorElement;
const $ul = Array.from(
document.querySelectorAll(".torrent-detail-page .list")
);
const [$firstColumn, $secondColumn] = $ul;
if (!$firstColumn || !$secondColumn) {
return { magnet: $a?.href };
}
const [_$category, _$type, _$language, $totalSize] = $firstColumn.children;
const [_$downloads, _$lastChecked, $dateUploaded] = $secondColumn.children;
return {
magnet: $a?.href,
fileSize: $totalSize.querySelector("span")!.textContent,
uploadDate: formatUploadDate(
$dateUploaded.querySelector("span")!.textContent!
),
};
};
export const getTorrentListLastPage = async (user: string) => {
const response = await request1337x(`/user/${user}/1`);
const { window } = new JSDOM(response);
const $ul = window.document.querySelector(".pagination > ul");
if ($ul) {
const $li = Array.from($ul.querySelectorAll("li")).at(-1);
const text = $li?.textContent;
if (text === ">>") {
const $previousLi = Array.from($ul.querySelectorAll("li")).at(-2);
return Number($previousLi?.textContent);
}
return Number(text);
}
return -1;
};
export const extractTorrentsFromDocument = async (
page: number,
user: string,
document: Document
) => {
const $trs = Array.from(document.querySelectorAll("tbody tr"));
return Promise.all(
$trs.map(async ($tr) => {
const $td = $tr.querySelector("td");
const [, $name] = Array.from($td!.querySelectorAll("a"));
const url = $name.href;
const title = $name.textContent ?? "";
const details = await getTorrentDetails(url);
return {
title,
magnet: details.magnet,
fileSize: details.fileSize ?? "N/A",
uploadDate: details.uploadDate ?? new Date(),
repacker: user,
page,
};
})
);
};
export const getNewRepacksFromUser = async (
user: string,
existingRepacks: Repack[],
page = 1
) => {
const response = await request1337x(`/user/${user}/${page}`);
const { window } = new JSDOM(response);
const repacks = await extractTorrentsFromDocument(
page,
user,
window.document
);
const newRepacks = repacks.filter(
(repack) =>
!existingRepacks.some(
(existingRepack) => existingRepack.title === repack.title
)
);
if (!newRepacks.length) return;
await savePage(newRepacks);
return getNewRepacksFromUser(user, existingRepacks, page + 1);
};

96
src/main/services/repack-tracker/gog.ts
View File

@@ -1,96 +0,0 @@
import { JSDOM, VirtualConsole } from "jsdom";
import { requestWebPage, savePage } from "./helpers";
import { Repack } from "@main/entity";
import { QueryDeepPartialEntity } from "typeorm/query-builder/QueryPartialEntity";
const virtualConsole = new VirtualConsole();
const getUploadDate = (document: Document) => {
const $modifiedTime = document.querySelector(
'[property="article:modified_time"]'
) as HTMLMetaElement;
if ($modifiedTime) return $modifiedTime.content;
const $publishedTime = document.querySelector(
'[property="article:published_time"]'
) as HTMLMetaElement;
return $publishedTime.content;
};
const getDownloadLink = (document: Document) => {
const $latestDownloadButton = document.querySelector(
".download-btn:not(.lightweight-accordion *)"
) as HTMLAnchorElement;
if ($latestDownloadButton) return $latestDownloadButton.href;
const $downloadButton = document.querySelector(
".download-btn"
) as HTMLAnchorElement;
if (!$downloadButton) return null;
return $downloadButton.href;
};
const getMagnet = (downloadLink: string) => {
if (downloadLink.startsWith("http")) {
const { searchParams } = new URL(downloadLink);
return Buffer.from(searchParams.get("url")!, "base64").toString("utf-8");
}
return downloadLink;
};
const getGOGGame = async (url: string) => {
const data = await requestWebPage(url);
const { window } = new JSDOM(data, { virtualConsole });
const downloadLink = getDownloadLink(window.document);
if (!downloadLink) return null;
const $em = window.document.querySelector("p em");
if (!$em) return null;
const fileSize = $em.textContent!.split("Size: ").at(1);
return {
fileSize: fileSize ?? "N/A",
uploadDate: new Date(getUploadDate(window.document)),
repacker: "GOG",
magnet: getMagnet(downloadLink),
page: 1,
};
};
export const getNewGOGGames = async (existingRepacks: Repack[] = []) => {
const data = await requestWebPage(
"https://freegogpcgames.com/a-z-games-list/"
);
const { window } = new JSDOM(data, { virtualConsole });
const $uls = Array.from(window.document.querySelectorAll(".az-columns"));
for (const $ul of $uls) {
const repacks: QueryDeepPartialEntity<Repack>[] = [];
const $lis = Array.from($ul.querySelectorAll("li"));
for (const $li of $lis) {
const $a = $li.querySelector("a")!;
const href = $a.href;
const title = $a.textContent!.trim();
const gameExists = existingRepacks.some(
(existingRepack) => existingRepack.title === title
);
if (!gameExists) {
const game = await getGOGGame(href);
if (game) repacks.push({ ...game, title });
}
}
if (repacks.length) await savePage(repacks);
}
};

40
src/main/services/repack-tracker/helpers.ts
View File

@@ -1,40 +0,0 @@
import axios from "axios";
import UserAgent from "user-agents";
import type { Repack } from "@main/entity";
import { repackRepository } from "@main/repository";
import { QueryDeepPartialEntity } from "typeorm/query-builder/QueryPartialEntity";
export const savePage = async (repacks: QueryDeepPartialEntity<Repack>[]) =>
Promise.all(
repacks.map((repack) => repackRepository.insert(repack).catch(() => {}))
);
export const requestWebPage = async (url: string) => {
const userAgent = new UserAgent();
return axios
.get(url, {
headers: {
"User-Agent": userAgent.toString(),
},
})
.then((response) => response.data);
};
export const decodeNonUtf8Response = async (res: Response) => {
const contentType = res.headers.get("content-type");
if (!contentType) return res.text();
const charset = contentType.substring(contentType.indexOf("charset=") + 8);
const text = await res.arrayBuffer().then((ab) => {
const dataView = new DataView(ab);
const decoder = new TextDecoder(charset);
return decoder.decode(dataView);
});
return text;
};

4
src/main/services/repack-tracker/index.ts
View File

@@ -1,4 +0,0 @@
export * from "./1337x";
export * from "./xatab";
export * from "./gog";
export * from "./online-fix";

157
src/main/services/repack-tracker/online-fix.ts
View File

@@ -1,157 +0,0 @@
import { Repack } from "@main/entity";
import { decodeNonUtf8Response, savePage } from "./helpers";
import { logger } from "../logger";
import { JSDOM } from "jsdom";
import createWorker from "@main/workers/torrent-parser.worker?nodeWorker";
import { toMagnetURI } from "parse-torrent";
const worker = createWorker({});
import makeFetchCookie from "fetch-cookie";
import { QueryDeepPartialEntity } from "typeorm/query-builder/QueryPartialEntity";
import { formatBytes } from "@shared";
const ONLINE_FIX_URL = "https://online-fix.me/";
let totalPages = 1;
export const getNewRepacksFromOnlineFix = async (
existingRepacks: Repack[] = [],
page = 1,
cookieJar = new makeFetchCookie.toughCookie.CookieJar()
): Promise<void> => {
const hasCredentials =
import.meta.env.MAIN_VITE_ONLINEFIX_USERNAME &&
import.meta.env.MAIN_VITE_ONLINEFIX_PASSWORD;
if (!hasCredentials) return;
const http = makeFetchCookie(fetch, cookieJar);
if (page === 1) {
await http(ONLINE_FIX_URL);
const preLogin =
((await http("https://online-fix.me/engine/ajax/authtoken.php", {
method: "GET",
headers: {
"X-Requested-With": "XMLHttpRequest",
Referer: ONLINE_FIX_URL,
},
}).then((res) => res.json())) as {
field: string;
value: string;
}) || undefined;
if (!preLogin.field || !preLogin.value) return;
const params = new URLSearchParams({
login_name: import.meta.env.MAIN_VITE_ONLINEFIX_USERNAME,
login_password: import.meta.env.MAIN_VITE_ONLINEFIX_PASSWORD,
login: "submit",
[preLogin.field]: preLogin.value,
});
await http(ONLINE_FIX_URL, {
method: "POST",
headers: {
Referer: ONLINE_FIX_URL,
Origin: ONLINE_FIX_URL,
"Content-Type": "application/x-www-form-urlencoded",
},
body: params.toString(),
});
}
const pageParams = page > 1 ? `${`/page/${page}`}` : "";
const home = await http(`https://online-fix.me${pageParams}`).then((res) =>
decodeNonUtf8Response(res)
);
const document = new JSDOM(home).window.document;
const repacks: QueryDeepPartialEntity<Repack>[] = [];
const articles = Array.from(document.querySelectorAll(".news"));
if (page == 1) {
totalPages = Number(
document.querySelector("nav > a:nth-child(13)")?.textContent
);
}
try {
await Promise.all(
articles.map(async (article) => {
const gameLink = article.querySelector("a")?.getAttribute("href");
if (!gameLink) return;
const gamePage = await http(gameLink).then((res) =>
decodeNonUtf8Response(res)
);
const gameDocument = new JSDOM(gamePage).window.document;
const torrentButtons = Array.from(
gameDocument.querySelectorAll("a")
).filter((a) => a.textContent?.includes("Torrent"));
const torrentPrePage = torrentButtons[0]?.getAttribute("href");
if (!torrentPrePage) return;
const torrentPage = await http(torrentPrePage, {
headers: {
Referer: gameLink,
},
}).then((res) => res.text());
const torrentDocument = new JSDOM(torrentPage).window.document;
const torrentLink = torrentDocument
.querySelector("a:nth-child(2)")
?.getAttribute("href");
const torrentFile = Buffer.from(
await http(`${torrentPrePage}${torrentLink}`).then((res) =>
res.arrayBuffer()
)
);
worker.once("message", (torrent) => {
if (!torrent) return;
const { name, created } = torrent;
repacks.push({
fileSize: formatBytes(torrent.length ?? 0),
magnet: toMagnetURI(torrent),
page: 1,
repacker: "onlinefix",
title: name,
uploadDate: created,
});
});
worker.postMessage(torrentFile);
})
);
} catch (err: unknown) {
logger.error((err as Error).message, {
method: "getNewRepacksFromOnlineFix",
});
}
const newRepacks = repacks.filter(
(repack) =>
repack.uploadDate &&
!existingRepacks.some(
(existingRepack) => existingRepack.title === repack.title
)
);
if (!newRepacks.length) return;
await savePage(newRepacks);
if (page === totalPages) return;
return getNewRepacksFromOnlineFix(existingRepacks, page + 1, cookieJar);
};

120
src/main/services/repack-tracker/xatab.ts
View File

@@ -1,120 +0,0 @@
import { JSDOM } from "jsdom";
import { Repack } from "@main/entity";
import { logger } from "../logger";
import { requestWebPage, savePage } from "./helpers";
import createWorker from "@main/workers/torrent-parser.worker?nodeWorker";
import { toMagnetURI } from "parse-torrent";
import type { Instance } from "parse-torrent";
import { QueryDeepPartialEntity } from "typeorm/query-builder/QueryPartialEntity";
import { formatBytes } from "@shared";
import { getFileBuffer } from "@main/helpers";
const worker = createWorker({});
worker.setMaxListeners(11);
let totalPages = 1;
const formatXatabDate = (str: string) => {
const date = new Date();
const [day, month, year] = str.split(".");
date.setDate(Number(day));
date.setMonth(Number(month) - 1);
date.setFullYear(Number(year));
date.setHours(0, 0, 0, 0);
return date;
};
const getXatabRepack = (
url: string
): Promise<{ fileSize: string; magnet: string; uploadDate: Date } | null> => {
return new Promise((resolve) => {
(async () => {
const data = await requestWebPage(url);
const { window } = new JSDOM(data);
const { document } = window;
const $uploadDate = document.querySelector(".entry__date");
const $downloadButton = document.querySelector(
".download-torrent"
) as HTMLAnchorElement;
if (!$downloadButton) return resolve(null);
worker.once("message", (torrent: Instance | null) => {
if (!torrent) return resolve(null);
resolve({
fileSize: formatBytes(torrent.length ?? 0),
magnet: toMagnetURI(torrent),
uploadDate: formatXatabDate($uploadDate!.textContent!),
});
});
const buffer = await getFileBuffer($downloadButton.href);
worker.postMessage(buffer);
})();
});
};
export const getNewRepacksFromXatab = async (
existingRepacks: Repack[] = [],
page = 1
): Promise<void> => {
const data = await requestWebPage(`https://byxatab.com/page/${page}`);
const { window } = new JSDOM(data);
const repacks: QueryDeepPartialEntity<Repack>[] = [];
if (page === 1) {
totalPages = Number(
window.document.querySelector(
"#bottom-nav > div.pagination > a:nth-child(12)"
)?.textContent
);
}
const repacksFromPage = Array.from(
window.document.querySelectorAll(".entry__title a")
).map(($a) => {
return getXatabRepack(($a as HTMLAnchorElement).href)
.then((repack) => {
if (repack) {
repacks.push({
title: $a.textContent!,
repacker: "Xatab",
...repack,
page,
});
}
})
.catch((err: unknown) => {
logger.error((err as Error).message, {
method: "getNewRepacksFromXatab",
});
});
});
await Promise.all(repacksFromPage);
const newRepacks = repacks.filter(
(repack) =>
!existingRepacks.some(
(existingRepack) => existingRepack.title === repack.title
)
);
if (!newRepacks.length) return;
await savePage(newRepacks);
if (page === totalPages) return;
return getNewRepacksFromXatab(existingRepacks, page + 1);
};

36
src/main/services/search-engine.ts Normal file
View File

@@ -0,0 +1,36 @@
import flexSearch from "flexsearch";
import { repackRepository } from "@main/repository";
import { formatName } from "@shared";
import type { GameRepack } from "@types";
export class SearchEngine {
public static repacks: GameRepack[] = [];
private static repacksIndex = new flexSearch.Index();
public static searchRepacks(query: string): GameRepack[] {
return this.repacksIndex
.search(formatName(query))
.map((index) => this.repacks[index]);
}
public static async updateRepacks() {
this.repacks = [];
const repacks = await repackRepository.find({
order: {
createdAt: "desc",
},
});
for (let i = 0; i < repacks.length; i++) {
const repack = repacks[i];
const formattedTitle = formatName(repack.title);
this.repacks = [...this.repacks, { ...repack, title: formattedTitle }];
this.repacksIndex.add(i, formattedTitle);
}
}
}

33
src/main/services/update-resolver.ts
View File

@@ -1,33 +0,0 @@
import path from "node:path";
import { app } from "electron";
import { chunk } from "lodash-es";
import { createDataSource } from "@main/data-source";
import { Repack } from "@main/entity";
import { repackRepository } from "@main/repository";
export const resolveDatabaseUpdates = async () => {
const updateDataSource = createDataSource({
database: app.isPackaged
? path.join(process.resourcesPath, "hydra.db")
: path.join(__dirname, "..", "..", "hydra.db"),
});
return updateDataSource.initialize().then(async () => {
const updateRepackRepository = updateDataSource.getRepository(Repack);
const updateRepacks = await updateRepackRepository.find();
const updateRepacksChunks = chunk(updateRepacks, 800);
for (const chunk of updateRepacksChunks) {
await repackRepository
.createQueryBuilder()
.insert()
.values(chunk)
.orIgnore()
.execute();
}
});
};