feat: migrating to electron-vite

src/main/services/repack-tracker/1337x.ts

@@ -0,0 +1,135 @@
+import { JSDOM } from "jsdom";
+
+import { formatUploadDate } from "@main/helpers";
+
+import { Repack } from "@main/entity";
+import { requestWebPage, savePage } from "./helpers";
+import type { GameRepackInput } from "./helpers";
+
+export const request1337x = async (path: string) =>
+  requestWebPage(`https://1337xx.to${path}`);
+
+/* TODO: $a will often be null */
+const getTorrentDetails = async (path: string) => {
+  const response = await request1337x(path);
+
+  const { window } = new JSDOM(response);
+  const { document } = window;
+
+  const $a = window.document.querySelector(
+    ".torrentdown1"
+  ) as HTMLAnchorElement;
+
+  const $ul = Array.from(
+    document.querySelectorAll(".torrent-detail-page .list")
+  );
+  const [$firstColumn, $secondColumn] = $ul;
+
+  if (!$firstColumn || !$secondColumn) {
+    return { magnet: $a?.href };
+  }
+
+  const [_$category, _$type, _$language, $totalSize] = $firstColumn.children;
+  const [_$downloads, _$lastChecked, $dateUploaded] = $secondColumn.children;
+
+  return {
+    magnet: $a?.href,
+    fileSize: $totalSize.querySelector("span").textContent ?? undefined,
+    uploadDate: formatUploadDate(
+      $dateUploaded.querySelector("span").textContent!
+    ),
+  };
+};
+
+export const getTorrentListLastPage = async (user: string) => {
+  const response = await request1337x(`/user/${user}/1`);
+
+  const { window } = new JSDOM(response);
+
+  const $ul = window.document.querySelector(".pagination > ul");
+
+  if ($ul) {
+    const $li = Array.from($ul.querySelectorAll("li")).at(-1);
+    const text = $li?.textContent;
+
+    if (text === ">>") {
+      const $previousLi = Array.from($ul.querySelectorAll("li")).at(-2);
+      return Number($previousLi?.textContent);
+    }
+
+    return Number(text);
+  }
+
+  return -1;
+};
+
+export const extractTorrentsFromDocument = async (
+  page: number,
+  user: string,
+  document: Document,
+  existingRepacks: Repack[] = []
+): Promise<GameRepackInput[]> => {
+  const $trs = Array.from(document.querySelectorAll("tbody tr"));
+
+  return Promise.all(
+    $trs.map(async ($tr) => {
+      const $td = $tr.querySelector("td");
+
+      const [, $name] = Array.from($td!.querySelectorAll("a"));
+      const url = $name.href;
+      const title = $name.textContent ?? "";
+
+      if (existingRepacks.some((repack) => repack.title === title)) {
+        return {
+          title,
+          magnet: "",
+          fileSize: null,
+          uploadDate: null,
+          repacker: user,
+          page,
+        };
+      }
+
+      const details = await getTorrentDetails(url);
+
+      return {
+        title,
+        magnet: details.magnet,
+        fileSize: details.fileSize ?? null,
+        uploadDate: details.uploadDate ?? null,
+        repacker: user,
+        page,
+      };
+    })
+  );
+};
+
+export const getNewRepacksFromUser = async (
+  user: string,
+  existingRepacks: Repack[],
+  page = 1
+): Promise<Repack[]> => {
+  const response = await request1337x(`/user/${user}/${page}`);
+  const { window } = new JSDOM(response);
+
+  const repacks = await extractTorrentsFromDocument(
+    page,
+    user,
+    window.document,
+    existingRepacks
+  );
+
+  const newRepacks = repacks.filter(
+    (repack) =>
+      repack.uploadDate &&
+      !existingRepacks.some(
+        (existingRepack) => existingRepack.title === repack.title
+      )
+  );
+
+  if (!newRepacks.length) return;
+
+  await savePage(newRepacks);
+
+  return getNewRepacksFromUser(user, existingRepacks, page + 1);
+};

src/main/services/repack-tracker/cpg-repacks.ts

@@ -0,0 +1,65 @@
+import { JSDOM } from "jsdom";
+
+import { Repack } from "@main/entity";
+
+import { requestWebPage, savePage } from "./helpers";
+import type { GameRepackInput } from "./helpers";
+import { logger } from "../logger";
+
+export const getNewRepacksFromCPG = async (
+  existingRepacks: Repack[] = [],
+  page = 1
+): Promise<void> => {
+  const data = await requestWebPage(`https://cpgrepacks.site/page/${page}`);
+
+  const { window } = new JSDOM(data);
+
+  const repacks: GameRepackInput[] = [];
+
+  try {
+    Array.from(window.document.querySelectorAll(".post")).forEach(($post) => {
+      const $title = $post.querySelector(".entry-title");
+      const uploadDate = $post.querySelector("time").getAttribute("datetime");
+
+      const $downloadInfo = Array.from(
+        $post.querySelectorAll(".wp-block-heading")
+      ).find(($heading) => $heading.textContent.startsWith("Download"));
+
+      /* Side note: CPG often misspells "Magnet" as "Magent" */
+      const $magnet = Array.from($post.querySelectorAll("a")).find(
+        ($a) =>
+          $a.textContent.startsWith("Magnet") ||
+          $a.textContent.startsWith("Magent")
+      );
+
+      const fileSize = $downloadInfo.textContent
+        .split("Download link => ")
+        .at(1);
+
+      repacks.push({
+        title: $title.textContent,
+        fileSize: fileSize ?? "N/A",
+        magnet: $magnet.href,
+        repacker: "CPG",
+        page,
+        uploadDate: new Date(uploadDate),
+      });
+    });
+  } catch (err) {
+    logger.error(err.message, { method: "getNewRepacksFromCPG" });
+  }
+
+  const newRepacks = repacks.filter(
+    (repack) =>
+      repack.uploadDate &&
+      !existingRepacks.some(
+        (existingRepack) => existingRepack.title === repack.title
+      )
+  );
+
+  if (!newRepacks.length) return;
+
+  await savePage(newRepacks);
+
+  return getNewRepacksFromCPG(existingRepacks, page + 1);
+};

src/main/services/repack-tracker/gog.ts

@@ -0,0 +1,78 @@
+import { JSDOM, VirtualConsole } from "jsdom";
+import { GameRepackInput, requestWebPage, savePage } from "./helpers";
+import { Repack } from "@main/entity";
+import { logger } from "../logger";
+
+const virtualConsole = new VirtualConsole();
+
+const getGOGGame = async (url: string) => {
+  const data = await requestWebPage(url);
+  const { window } = new JSDOM(data, { virtualConsole });
+
+  const $modifiedTime = window.document.querySelector(
+    '[property="article:modified_time"]'
+  ) as HTMLMetaElement;
+
+  const $em = window.document.querySelector(
+    "p:not(.lightweight-accordion *) em"
+  );
+  const fileSize = $em.textContent.split("Size: ").at(1);
+  const $downloadButton = window.document.querySelector(
+    ".download-btn:not(.lightweight-accordion *)"
+  ) as HTMLAnchorElement;
+
+  const { searchParams } = new URL($downloadButton.href);
+  const magnet = Buffer.from(searchParams.get("url"), "base64").toString(
+    "utf-8"
+  );
+
+  return {
+    fileSize: fileSize ?? "N/A",
+    uploadDate: new Date($modifiedTime.content),
+    repacker: "GOG",
+    magnet,
+    page: 1,
+  };
+};
+
+export const getNewGOGGames = async (existingRepacks: Repack[] = []) => {
+  try {
+    const data = await requestWebPage(
+      "https://freegogpcgames.com/a-z-games-list/"
+    );
+
+    const { window } = new JSDOM(data, { virtualConsole });
+
+    const $uls = Array.from(window.document.querySelectorAll(".az-columns"));
+
+    for (const $ul of $uls) {
+      const repacks: GameRepackInput[] = [];
+      const $lis = Array.from($ul.querySelectorAll("li"));
+
+      for (const $li of $lis) {
+        const $a = $li.querySelector("a");
+        const href = $a.href;
+
+        const title = $a.textContent.trim();
+
+        const gameExists = existingRepacks.some(
+          (existingRepack) => existingRepack.title === title
+        );
+
+        if (!gameExists) {
+          try {
+            const game = await getGOGGame(href);
+
+            repacks.push({ ...game, title });
+          } catch (err) {
+            logger.error(err.message, { method: "getGOGGame", url: href });
+          }
+        }
+      }
+
+      if (repacks.length) await savePage(repacks);
+    }
+  } catch (err) {
+    logger.error(err.message, { method: "getNewGOGGames" });
+  }
+};

src/main/services/repack-tracker/helpers.ts

@@ -0,0 +1,18 @@
+import { repackRepository } from "@main/repository";
+
+import type { GameRepack } from "@types";
+
+export type GameRepackInput = Omit<
+  GameRepack,
+  "id" | "repackerFriendlyName" | "createdAt" | "updatedAt"
+>;
+
+export const savePage = async (repacks: GameRepackInput[]) =>
+  Promise.all(
+    repacks.map((repack) => repackRepository.insert(repack).catch(() => {}))
+  );
+
+export const requestWebPage = async (url: string) =>
+  fetch(url, {
+    method: "GET",
+  }).then((response) => response.text());

src/main/services/repack-tracker/index.ts

@@ -0,0 +1,5 @@
+export * from "./1337x";
+export * from "./xatab";
+export * from "./cpg-repacks";
+export * from "./gog";
+// export * from "./online-fix";

src/main/services/repack-tracker/online-fix.ts

@@ -0,0 +1,207 @@
+import { Repack } from "@main/entity";
+import { savePage } from "./helpers";
+import type { GameRepackInput } from "./helpers";
+import { logger } from "../logger";
+import parseTorrent, {
+  toMagnetURI,
+  Instance as TorrentInstance,
+} from "parse-torrent";
+import { JSDOM } from "jsdom";
+import { gotScraping } from "got-scraping";
+import { CookieJar } from "tough-cookie";
+
+import { format, parse, sub } from "date-fns";
+import { ru } from "date-fns/locale";
+import { decode } from "windows-1251";
+import { onlinefixFormatter } from "@main/helpers";
+
+export const getNewRepacksFromOnlineFix = async (
+  existingRepacks: Repack[] = [],
+  page = 1,
+  cookieJar = new CookieJar()
+): Promise<void> => {
+  const hasCredentials =
+    import.meta.env.MAIN_VITE_ONLINEFIX_USERNAME &&
+    import.meta.env.MAIN_VITE_ONLINEFIX_PASSWORD;
+  if (!hasCredentials) return;
+
+  const http = gotScraping.extend({
+    headerGeneratorOptions: {
+      browsers: [
+        {
+          name: "chrome",
+          minVersion: 87,
+          maxVersion: 89,
+        },
+      ],
+      devices: ["desktop"],
+      locales: ["en-US"],
+      operatingSystems: ["windows", "linux"],
+    },
+    cookieJar: cookieJar,
+  });
+
+  if (page === 1) {
+    await http.get("https://online-fix.me/");
+    const preLogin =
+      ((await http
+        .get("https://online-fix.me/engine/ajax/authtoken.php", {
+          headers: {
+            "X-Requested-With": "XMLHttpRequest",
+            Referer: "https://online-fix.me/",
+          },
+        })
+        .json()) as {
+        field: string;
+        value: string;
+      }) || undefined;
+
+    if (!preLogin.field || !preLogin.value) return;
+
+    const params = new URLSearchParams({
+      login_name: import.meta.env.MAIN_VITE_ONLINEFIX_USERNAME,
+      login_password: import.meta.env.MAIN_VITE_ONLINEFIX_PASSWORD,
+      login: "submit",
+      [preLogin.field]: preLogin.value,
+    });
+
+    await http
+      .post("https://online-fix.me/", {
+        encoding: "binary",
+        headers: {
+          Referer: "https://online-fix.me",
+          Origin: "https://online-fix.me",
+          "Content-Type": "application/x-www-form-urlencoded",
+        },
+        body: params.toString(),
+      })
+      .text();
+  }
+
+  const pageParams = page > 1 ? `${`/page/${page}`}` : "";
+
+  const home = await http.get(`https://online-fix.me${pageParams}`, {
+    encoding: "binary",
+  });
+  const document = new JSDOM(home.body).window.document;
+
+  const repacks: GameRepackInput[] = [];
+  const articles = Array.from(document.querySelectorAll(".news"));
+  const totalPages = Number(
+    document.querySelector("nav > a:nth-child(13)").textContent
+  );
+
+  try {
+    await Promise.all(
+      articles.map(async (article) => {
+        const gameName = onlinefixFormatter(
+          decode(article.querySelector("h2.title")?.textContent?.trim())
+        );
+
+        const gameLink = article.querySelector("a")?.getAttribute("href");
+
+        if (!gameLink) return;
+
+        const gamePage = await http
+          .get(gameLink, {
+            encoding: "binary",
+          })
+          .text();
+
+        const gameDocument = new JSDOM(gamePage).window.document;
+
+        const uploadDateText = gameDocument.querySelector("time").textContent;
+
+        let decodedDateText = decode(uploadDateText);
+
+        // "Вчера" means yesterday.
+        if (decodedDateText.includes("Вчера")) {
+          const yesterday = sub(new Date(), { days: 1 });
+          const formattedYesterday = format(yesterday, "d LLLL yyyy", {
+            locale: ru,
+          });
+          decodedDateText = decodedDateText.replace(
+            "Вчера", // "Change yesterday to the default expected date format"
+            formattedYesterday
+          );
+        }
+
+        const uploadDate = parse(
+          decodedDateText,
+          "d LLLL yyyy, HH:mm",
+          new Date(),
+          {
+            locale: ru,
+          }
+        );
+
+        const torrentButtons = Array.from(
+          gameDocument.querySelectorAll("a")
+        ).filter((a) => a.textContent?.includes("Torrent"));
+
+        const torrentPrePage = torrentButtons[0]?.getAttribute("href");
+        if (!torrentPrePage) return;
+
+        const torrentPage = await http
+          .get(torrentPrePage, {
+            encoding: "binary",
+            headers: {
+              Referer: gameLink,
+            },
+          })
+          .text();
+
+        const torrentDocument = new JSDOM(torrentPage).window.document;
+
+        const torrentLink = torrentDocument
+          .querySelector("a:nth-child(2)")
+          ?.getAttribute("href");
+
+        const torrentFile = Buffer.from(
+          await http
+            .get(`${torrentPrePage}/${torrentLink}`, {
+              responseType: "buffer",
+            })
+            .buffer()
+        );
+
+        const torrent = parseTorrent(torrentFile) as TorrentInstance;
+        const magnetLink = toMagnetURI({
+          infoHash: torrent.infoHash,
+        });
+
+        const torrentSizeInBytes = torrent.length;
+        const fileSizeFormatted =
+          torrentSizeInBytes >= 1024 ** 3
+            ? `${(torrentSizeInBytes / 1024 ** 3).toFixed(1)}GBs`
+            : `${(torrentSizeInBytes / 1024 ** 2).toFixed(1)}MBs`;
+
+        repacks.push({
+          fileSize: fileSizeFormatted,
+          magnet: magnetLink,
+          page: 1,
+          repacker: "onlinefix",
+          title: gameName,
+          uploadDate: uploadDate,
+        });
+      })
+    );
+  } catch (err) {
+    logger.error(err.message, { method: "getNewRepacksFromOnlineFix" });
+  }
+
+  const newRepacks = repacks.filter(
+    (repack) =>
+      repack.uploadDate &&
+      !existingRepacks.some(
+        (existingRepack) => existingRepack.title === repack.title
+      )
+  );
+
+  if (!newRepacks.length) return;
+  if (page === totalPages) return;
+
+  await savePage(newRepacks);
+
+  return getNewRepacksFromOnlineFix(existingRepacks, page + 1, cookieJar);
+};

src/main/services/repack-tracker/xatab.ts

@@ -0,0 +1,95 @@
+import { JSDOM } from "jsdom";
+
+import parseTorrent, { toMagnetURI } from "parse-torrent";
+
+import { Repack } from "@main/entity";
+import { logger } from "../logger";
+import { requestWebPage, savePage } from "./helpers";
+import type { GameRepackInput } from "./helpers";
+
+const getTorrentBuffer = (url: string) =>
+  fetch(url, { method: "GET" }).then((response) =>
+    response.arrayBuffer().then((buffer) => Buffer.from(buffer))
+  );
+
+const formatXatabDate = (str: string) => {
+  const date = new Date();
+
+  const [day, month, year] = str.split(".");
+
+  date.setDate(Number(day));
+  date.setMonth(Number(month) - 1);
+  date.setFullYear(Number(year));
+  date.setHours(0, 0, 0, 0);
+
+  return date;
+};
+
+const formatXatabDownloadSize = (str: string) =>
+  str.replace(",", ".").replace(/Гб/g, "GB").replace(/Мб/g, "MB");
+
+const getXatabRepack = async (url: string) => {
+  const data = await requestWebPage(url);
+  const { window } = new JSDOM(data);
+
+  const $uploadDate = window.document.querySelector(".entry__date");
+  const $size = window.document.querySelector(".entry__info-size");
+
+  const $downloadButton = window.document.querySelector(
+    ".download-torrent"
+  ) as HTMLAnchorElement;
+
+  if (!$downloadButton) throw new Error("Download button not found");
+
+  const torrentBuffer = await getTorrentBuffer($downloadButton.href);
+
+  return {
+    fileSize: formatXatabDownloadSize($size.textContent).toUpperCase(),
+    magnet: toMagnetURI({
+      infoHash: parseTorrent(torrentBuffer).infoHash,
+    }),
+    uploadDate: formatXatabDate($uploadDate.textContent),
+  };
+};
+
+export const getNewRepacksFromXatab = async (
+  existingRepacks: Repack[] = [],
+  page = 1
+): Promise<void> => {
+  const data = await requestWebPage(`https://byxatab.com/page/${page}`);
+
+  const { window } = new JSDOM(data);
+
+  const repacks: GameRepackInput[] = [];
+
+  for (const $a of Array.from(
+    window.document.querySelectorAll(".entry__title a")
+  )) {
+    try {
+      const repack = await getXatabRepack(($a as HTMLAnchorElement).href);
+
+      repacks.push({
+        title: $a.textContent,
+        repacker: "Xatab",
+        ...repack,
+        page,
+      });
+    } catch (err) {
+      logger.error(err.message, { method: "getNewRepacksFromXatab" });
+    }
+  }
+
+  const newRepacks = repacks.filter(
+    (repack) =>
+      repack.uploadDate &&
+      !existingRepacks.some(
+        (existingRepack) => existingRepack.title === repack.title
+      )
+  );
+
+  if (!newRepacks.length) return;
+
+  await savePage(newRepacks);
+
+  return getNewRepacksFromXatab(existingRepacks, page + 1);
+};