first commit

src/main/services/repack-tracker/1337x.ts

@@ -0,0 +1,135 @@
+import { JSDOM } from "jsdom";
+
+import { formatUploadDate } from "@main/helpers";
+
+import { Repack } from "@main/entity";
+import { requestWebPage, savePage } from "./helpers";
+import type { GameRepackInput } from "./helpers";
+
+export const request1337x = async (path: string) =>
+  requestWebPage(`https://1337xx.to${path}`);
+
+/* TODO: $a will often be null */
+const getTorrentDetails = async (path: string) => {
+  const response = await request1337x(path);
+
+  const { window } = new JSDOM(response);
+  const { document } = window;
+
+  const $a = window.document.querySelector(
+    ".torrentdown1"
+  ) as HTMLAnchorElement;
+
+  const $ul = Array.from(
+    document.querySelectorAll(".torrent-detail-page .list")
+  );
+  const [$firstColumn, $secondColumn] = $ul;
+
+  if (!$firstColumn || !$secondColumn) {
+    return { magnet: $a?.href };
+  }
+
+  const [_$category, _$type, _$language, $totalSize] = $firstColumn.children;
+  const [_$downloads, _$lastChecked, $dateUploaded] = $secondColumn.children;
+
+  return {
+    magnet: $a?.href,
+    fileSize: $totalSize.querySelector("span").textContent ?? undefined,
+    uploadDate: formatUploadDate(
+      $dateUploaded.querySelector("span").textContent!
+    ),
+  };
+};
+
+export const getTorrentListLastPage = async (user: string) => {
+  const response = await request1337x(`/user/${user}/1`);
+
+  const { window } = new JSDOM(response);
+
+  const $ul = window.document.querySelector(".pagination > ul");
+
+  if ($ul) {
+    const $li = Array.from($ul.querySelectorAll("li")).at(-1);
+    const text = $li?.textContent;
+
+    if (text === ">>") {
+      const $previousLi = Array.from($ul.querySelectorAll("li")).at(-2);
+      return Number($previousLi?.textContent);
+    }
+
+    return Number(text);
+  }
+
+  return -1;
+};
+
+export const extractTorrentsFromDocument = async (
+  page: number,
+  user: string,
+  document: Document,
+  existingRepacks: Repack[] = []
+): Promise<GameRepackInput[]> => {
+  const $trs = Array.from(document.querySelectorAll("tbody tr"));
+
+  return Promise.all(
+    $trs.map(async ($tr) => {
+      const $td = $tr.querySelector("td");
+
+      const [, $name] = Array.from($td!.querySelectorAll("a"));
+      const url = $name.href;
+      const title = $name.textContent ?? "";
+
+      if (existingRepacks.some((repack) => repack.title === title)) {
+        return {
+          title,
+          magnet: "",
+          fileSize: null,
+          uploadDate: null,
+          repacker: user,
+          page,
+        };
+      }
+
+      const details = await getTorrentDetails(url);
+
+      return {
+        title,
+        magnet: details.magnet,
+        fileSize: details.fileSize ?? null,
+        uploadDate: details.uploadDate ?? null,
+        repacker: user,
+        page,
+      };
+    })
+  );
+};
+
+export const getNewRepacksFromUser = async (
+  user: string,
+  existingRepacks: Repack[],
+  page = 1
+): Promise<Repack[]> => {
+  const response = await request1337x(`/user/${user}/${page}`);
+  const { window } = new JSDOM(response);
+
+  const repacks = await extractTorrentsFromDocument(
+    page,
+    user,
+    window.document,
+    existingRepacks
+  );
+
+  const newRepacks = repacks.filter(
+    (repack) =>
+      repack.uploadDate &&
+      !existingRepacks.some(
+        (existingRepack) => existingRepack.title === repack.title
+      )
+  );
+
+  if (!newRepacks.length) return;
+
+  await savePage(newRepacks);
+
+  return getNewRepacksFromUser(user, existingRepacks, page + 1);
+};

src/main/services/repack-tracker/cpg-repacks.ts

@@ -0,0 +1,65 @@
+import { JSDOM } from "jsdom";
+
+import { Repack } from "@main/entity";
+
+import { requestWebPage, savePage } from "./helpers";
+import type { GameRepackInput } from "./helpers";
+import { logger } from "../logger";
+
+export const getNewRepacksFromCPG = async (
+  existingRepacks: Repack[] = [],
+  page = 1
+): Promise<void> => {
+  const data = await requestWebPage(`https://cpgrepacks.site/page/${page}`);
+
+  const { window } = new JSDOM(data);
+
+  const repacks: GameRepackInput[] = [];
+
+  try {
+    Array.from(window.document.querySelectorAll(".post")).forEach(($post) => {
+      const $title = $post.querySelector(".entry-title");
+      const uploadDate = $post.querySelector("time").getAttribute("datetime");
+
+      const $downloadInfo = Array.from(
+        $post.querySelectorAll(".wp-block-heading")
+      ).find(($heading) => $heading.textContent.startsWith("Download"));
+
+      /* Side note: CPG often misspells "Magnet" as "Magent" */
+      const $magnet = Array.from($post.querySelectorAll("a")).find(
+        ($a) =>
+          $a.textContent.startsWith("Magnet") ||
+          $a.textContent.startsWith("Magent")
+      );
+
+      const fileSize = $downloadInfo.textContent
+        .split("Download link => ")
+        .at(1);
+
+      repacks.push({
+        title: $title.textContent,
+        fileSize: fileSize ?? "N/A",
+        magnet: $magnet.href,
+        repacker: "CPG",
+        page,
+        uploadDate: new Date(uploadDate),
+      });
+    });
+  } catch (err) {
+    logger.error(err.message, { method: "getNewRepacksFromCPG" });
+  }
+
+  const newRepacks = repacks.filter(
+    (repack) =>
+      repack.uploadDate &&
+      !existingRepacks.some(
+        (existingRepack) => existingRepack.title === repack.title
+      )
+  );
+
+  if (!newRepacks.length) return;
+
+  await savePage(newRepacks);
+
+  return getNewRepacksFromCPG(existingRepacks, page + 1);
+};

src/main/services/repack-tracker/gog.ts

@@ -0,0 +1,78 @@
+import { JSDOM, VirtualConsole } from "jsdom";
+import { GameRepackInput, requestWebPage, savePage } from "./helpers";
+import { Repack } from "@main/entity";
+import { logger } from "../logger";
+
+const virtualConsole = new VirtualConsole();
+
+const getGOGGame = async (url: string) => {
+  const data = await requestWebPage(url);
+  const { window } = new JSDOM(data, { virtualConsole });
+
+  const $modifiedTime = window.document.querySelector(
+    '[property="article:modified_time"]'
+  ) as HTMLMetaElement;
+
+  const $em = window.document.querySelector(
+    "p:not(.lightweight-accordion *) em"
+  );
+  const fileSize = $em.textContent.split("Size: ").at(1);
+  const $downloadButton = window.document.querySelector(
+    ".download-btn:not(.lightweight-accordion *)"
+  ) as HTMLAnchorElement;
+
+  const { searchParams } = new URL($downloadButton.href);
+  const magnet = Buffer.from(searchParams.get("url"), "base64").toString(
+    "utf-8"
+  );
+
+  return {
+    fileSize: fileSize ?? "N/A",
+    uploadDate: new Date($modifiedTime.content),
+    repacker: "GOG",
+    magnet,
+    page: 1,
+  };
+};
+
+export const getNewGOGGames = async (existingRepacks: Repack[] = []) => {
+  try {
+    const data = await requestWebPage(
+      "https://freegogpcgames.com/a-z-games-list/"
+    );
+
+    const { window } = new JSDOM(data, { virtualConsole });
+
+    const $uls = Array.from(window.document.querySelectorAll(".az-columns"));
+
+    for (const $ul of $uls) {
+      const repacks: GameRepackInput[] = [];
+      const $lis = Array.from($ul.querySelectorAll("li"));
+
+      for (const $li of $lis) {
+        const $a = $li.querySelector("a");
+        const href = $a.href;
+
+        const title = $a.textContent.trim();
+
+        const gameExists = existingRepacks.some(
+          (existingRepack) => existingRepack.title === title
+        );
+
+        if (!gameExists) {
+          try {
+            const game = await getGOGGame(href);
+
+            repacks.push({ ...game, title });
+          } catch (err) {
+            logger.error(err.message, { method: "getGOGGame", url: href });
+          }
+        }
+      }
+
+      if (repacks.length) await savePage(repacks);
+    }
+  } catch (err) {
+    logger.error(err.message, { method: "getNewGOGGames" });
+  }
+};

src/main/services/repack-tracker/helpers.ts

@@ -0,0 +1,18 @@
+import { repackRepository } from "@main/repository";
+
+import type { GameRepack } from "@types";
+
+export type GameRepackInput = Omit<
+  GameRepack,
+  "id" | "repackerFriendlyName" | "createdAt" | "updatedAt"
+>;
+
+export const savePage = async (repacks: GameRepackInput[]) =>
+  Promise.all(
+    repacks.map((repack) => repackRepository.insert(repack).catch(() => {}))
+  );
+
+export const requestWebPage = async (url: string) =>
+  fetch(url, {
+    method: "GET",
+  }).then((response) => response.text());

src/main/services/repack-tracker/index.ts

@@ -0,0 +1,4 @@
+export * from "./1337x";
+export * from "./xatab";
+export * from "./cpg-repacks";
+export * from "./gog";

src/main/services/repack-tracker/xatab.ts

@@ -0,0 +1,95 @@
+import { JSDOM } from "jsdom";
+
+import parseTorrent, { toMagnetURI } from "parse-torrent";
+
+import { Repack } from "@main/entity";
+import { logger } from "../logger";
+import { requestWebPage, savePage } from "./helpers";
+import type { GameRepackInput } from "./helpers";
+
+const getTorrentBuffer = (url: string) =>
+  fetch(url, { method: "GET" }).then((response) =>
+    response.arrayBuffer().then((buffer) => Buffer.from(buffer))
+  );
+
+const formatXatabDate = (str: string) => {
+  const date = new Date();
+
+  const [day, month, year] = str.split(".");
+
+  date.setDate(Number(day));
+  date.setMonth(Number(month) - 1);
+  date.setFullYear(Number(year));
+  date.setHours(0, 0, 0, 0);
+
+  return date;
+};
+
+const formatXatabDownloadSize = (str: string) =>
+  str.replace(",", ".").replace(/Гб/g, "GB").replace(/Мб/g, "MB");
+
+const getXatabRepack = async (url: string) => {
+  const data = await requestWebPage(url);
+  const { window } = new JSDOM(data);
+
+  const $uploadDate = window.document.querySelector(".entry__date");
+  const $size = window.document.querySelector(".entry__info-size");
+
+  const $downloadButton = window.document.querySelector(
+    ".download-torrent"
+  ) as HTMLAnchorElement;
+
+  if (!$downloadButton) throw new Error("Download button not found");
+
+  const torrentBuffer = await getTorrentBuffer($downloadButton.href);
+
+  return {
+    fileSize: formatXatabDownloadSize($size.textContent).toUpperCase(),
+    magnet: toMagnetURI({
+      infoHash: parseTorrent(torrentBuffer).infoHash,
+    }),
+    uploadDate: formatXatabDate($uploadDate.textContent),
+  };
+};
+
+export const getNewRepacksFromXatab = async (
+  existingRepacks: Repack[] = [],
+  page = 1
+): Promise<void> => {
+  const data = await requestWebPage(`https://byxatab.com/page/${page}`);
+
+  const { window } = new JSDOM(data);
+
+  const repacks: GameRepackInput[] = [];
+
+  for (const $a of Array.from(
+    window.document.querySelectorAll(".entry__title a")
+  )) {
+    try {
+      const repack = await getXatabRepack(($a as HTMLAnchorElement).href);
+
+      repacks.push({
+        title: $a.textContent,
+        repacker: "Xatab",
+        ...repack,
+        page,
+      });
+    } catch (err) {
+      logger.error(err.message, { method: "getNewRepacksFromXatab" });
+    }
+  }
+
+  const newRepacks = repacks.filter(
+    (repack) =>
+      repack.uploadDate &&
+      !existingRepacks.some(
+        (existingRepack) => existingRepack.title === repack.title
+      )
+  );
+
+  if (!newRepacks.length) return;
+
+  await savePage(newRepacks);
+
+  return getNewRepacksFromXatab(existingRepacks, page + 1);
+};