From 8b1b8aa05c9eaf5409bf0687cff199e9c58ba111 Mon Sep 17 00:00:00 2001 From: David Ludwig Date: Tue, 22 Jun 2021 08:55:38 -0500 Subject: [PATCH] Remove old torrent handling code in Seeker --- services/seeker/src/torrents/Torrent.ts | 63 -------- services/seeker/src/torrents/index.ts | 5 - services/seeker/src/torrents/parsing.ts | 153 ------------------ .../seeker/src/torrents/providers/Provider.ts | 25 --- .../torrents/providers/torrentgalaxy/index.ts | 30 ---- .../providers/torrentgalaxy/search.ts | 135 ---------------- services/seeker/src/torrents/ranking.ts | 58 ------- services/seeker/src/torrents/util.ts | 82 ---------- 8 files changed, 551 deletions(-) delete mode 100644 services/seeker/src/torrents/Torrent.ts delete mode 100644 services/seeker/src/torrents/index.ts delete mode 100644 services/seeker/src/torrents/parsing.ts delete mode 100644 services/seeker/src/torrents/providers/Provider.ts delete mode 100644 services/seeker/src/torrents/providers/torrentgalaxy/index.ts delete mode 100644 services/seeker/src/torrents/providers/torrentgalaxy/search.ts delete mode 100644 services/seeker/src/torrents/ranking.ts delete mode 100644 services/seeker/src/torrents/util.ts diff --git a/services/seeker/src/torrents/Torrent.ts b/services/seeker/src/torrents/Torrent.ts deleted file mode 100644 index cd0c8e4..0000000 --- a/services/seeker/src/torrents/Torrent.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { MovieTicket } from "@autoplex/database"; -import { ITorrentMetaInfo, parseMovieTorrentName } from "./parsing"; - -export default class Torrent -{ - /** - * The name of the torrent - */ - public readonly name: string; - - /** - * The size of the torrent in bytes (if available) - */ - public readonly size: number | null; - - /** - * The number of seeders (if available) - */ - public readonly seeders: number; - - /** - * Download link (if available) - */ - protected readonly link: string | null; - - /** - * Metadata of the torrent - */ - public readonly metadata: ITorrentMetaInfo; - - /** - * Create a new Torrent instance - * - * @param name The name of the torrent - * @param size The size of the torrent in bytes (if available) - * @param seeders The number of seeders (if available) - * @param link The number of seeders (if available) - */ - public constructor(movie: MovieTicket, name: string, size?: number, seeders?: number, link?: string) { - this.name = name.trim(); - this.size = size ?? null; - this.seeders = seeders ?? 1; - this.link = link ?? null; - this.metadata = parseMovieTorrentName(name, movie.title ?? "", movie.year ?? undefined); - } - - /** - * Return a link to download (magnet or .torrent) - */ - public async downloadLink() { - if (this.link === null) { - throw Error("Magnet link does not exist"); - } - return this.link; - } - - /** - * Serialize this torrent into a string - */ - public toString() { - return `Name: ${this.name}; Size: ${this.size}; Seeders: ${this.seeders};` - } -} diff --git a/services/seeker/src/torrents/index.ts b/services/seeker/src/torrents/index.ts deleted file mode 100644 index da632a9..0000000 --- a/services/seeker/src/torrents/index.ts +++ /dev/null @@ -1,5 +0,0 @@ -import TorrentGalaxy from "./providers/torrentgalaxy"; - -export { - TorrentGalaxy -} diff --git a/services/seeker/src/torrents/parsing.ts b/services/seeker/src/torrents/parsing.ts deleted file mode 100644 index 80e40b5..0000000 --- a/services/seeker/src/torrents/parsing.ts +++ /dev/null @@ -1,153 +0,0 @@ -/** - * Video quality from lowest to highest - */ -export enum Resolution { - HD4k, - HD1080, - HD720, - SD384, - SD480, - SD360, - Unknown -} - -// https://en.wikipedia.org/wiki/Pirated_movie_release_types#DVD_and_VOD_ripping -// https://en.wikipedia.org/wiki/Standard_(warez)#cite_note-txd2k9-13 -/** - * Types of releases from lowest quality to highest - */ -export enum ReleaseType { - BluRay, - WebDl, - WebRip, - WebCap, - HDRip, - DVDR, - DVDRip, - Unknown, // Unknown is better than cam tbh - HDCAM, - CAM -} - -export enum VideoCodec { - XviD, - x264, - x265, -} - -export enum VideoCodecFlag { - REMUX, - HDR, - HEVC -} - -export enum AudioCodec { - AC3, - DD51, - AAC71, - Atmos71, - TenBit -} - -export interface ITorrentMetaInfo { - containsOtherLanguage: boolean, - resolution: Resolution, - releaseType: ReleaseType, -} - -/** - * Determine meta-info from a torrent name - */ -export function parseMovieTorrentName(torrentName: string, title: string = "", year?: number) { - // Split the meta info after the year if possible to make parsing more reliable - let split = torrentName.split(new RegExp(`${year}|\\(${year}\\)`)); - let metaInfo = split[split.length - 1]; - title = split.length > 1 ? "" : title; // No need to check title in parsing if split correctly - return { - containsOtherLanguage: determineIfContainsOtherLanguages(torrentName, title), - resolution: determineResolution(metaInfo, title), - releaseType: determineReleaseType(metaInfo, title), - } -} - -/** - * Examine the torrent name for language indicators - */ -function determineIfContainsOtherLanguages(torrentName: string, title: string) { - let matches = torrentName.match(/\b(?:Hindi|Telugu|Ita|Italian|Spanish|Latino|Russian|Arabic|Dual|Multi)\b/gi); - for (let match of matches ?? []) { - if (title.indexOf(match) == -1) { - return true; - } - } - return false; -} - -/** - * Interpret the resolution string as an enum value - */ -function resolutionFromString(resolution: string) { - switch(resolution.toUpperCase()) { - case "4K": - case "UHD": - case "2160": - return Resolution.HD4k; - case "1080": - return Resolution.HD1080; - case "720": - return Resolution.HD720; - case "480": - return Resolution.SD480; - case "384": - return Resolution.SD384; - case "360": - return Resolution.SD360; - default: - return Resolution.Unknown; - } -} - -/** - * Determine the video resolution of the torrent - */ -function determineResolution(torrentName: string, title: string) { - let matches = torrentName.match(/\b(?:2160|1080|720|480|384|360)p?|UltraHD|UHD|4K\b/gi); - if (matches == null) { - return Resolution.Unknown; - } - let resolution = matches[matches.length - 1]; - - // Make sure what was matched is not part of the title... - if (matches.length == 1 && title.indexOf(resolution) != -1) { - return Resolution.Unknown; - } - return resolutionFromString(resolution.replace(/p$/i, "")); -} - -/** - * Determine the release type of the torrent - */ -function determineReleaseType(torrentName: string, title: string) { - let releaseTypeRegex: {[type: string]: RegExp} = { - [ReleaseType.BluRay]: /\b(?:BR|Blu-Ray|BluRay|BDRip|BRRip|BDMV|BDR|BD25|BD50|BD5|BD9)\b/i, - [ReleaseType.WebDl] : /\b(?:WEB.?DL|WEB-DLRip)\b/i, - [ReleaseType.WebRip]: /\b(?:WEB.?Rip|WEB)\b/i, - [ReleaseType.WebCap]: /\bWEB.?Cap\b/i, - [ReleaseType.HDRip] : /\b(?:HC|HD.?Rip)\b/i, - [ReleaseType.DVDR] : /\bDVD.?R|DVD-Full|Full-Rip|DVD.?5|DVD.?9\b/i, - [ReleaseType.DVDRip]: /\bDVD.?Rip|DVD.?Mux/i, - [ReleaseType.HDCAM] : /\b(?:TRUE|HD)CAM\b/i, - [ReleaseType.CAM] : /\bCAM.?Rip\b/i, - }; - let matches: RegExpMatchArray | null; - for (let type in releaseTypeRegex) { - matches = torrentName.match(releaseTypeRegex[type]); - if (!matches) { - continue; - } - if (matches.length == 1 || title.indexOf(matches[matches.length - 1]) == -1) { - return parseInt(type); - } - } - return ReleaseType.Unknown; -} diff --git a/services/seeker/src/torrents/providers/Provider.ts b/services/seeker/src/torrents/providers/Provider.ts deleted file mode 100644 index 8eaa8d5..0000000 --- a/services/seeker/src/torrents/providers/Provider.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { MovieTicket } from "@autoplex/database"; -import Torrent from "../Torrent"; - -/** - * Media type flags - */ -export enum MediaType { - None = 0x0, - Movies = 0x1, - TvShows = 0x2 -} - -export default abstract class Provider -{ - /** - * Indicate what media types the provider supports - */ - public static readonly PROVIDES: MediaType = MediaType.None; - - /** - * Search for movies - */ - public abstract searchMovie(movie: MovieTicket): Promise; - -} diff --git a/services/seeker/src/torrents/providers/torrentgalaxy/index.ts b/services/seeker/src/torrents/providers/torrentgalaxy/index.ts deleted file mode 100644 index 8a08bcd..0000000 --- a/services/seeker/src/torrents/providers/torrentgalaxy/index.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { MovieTicket } from "@autoplex/database"; -import Provider, { MediaType } from "../Provider"; -import Torrent from "../../Torrent"; -import { search, Sort } from "./search"; - - -export default class TorrentGalaxy extends Provider -{ - /** - * Indicate that this provider provides movies - */ - public static readonly PROVIDES = MediaType.Movies; - - /** - * Search for a movie - */ - public async searchMovie(movie: MovieTicket) { - if (movie.imdbId === null) { - return []; - } - let torrents = await search(movie.imdbId, undefined, Sort.Seeders); - return torrents.torrents.map(torrent => new Torrent( - movie, - torrent.name, - torrent.size, - torrent.seeders, - torrent.magnet - )); - } -} diff --git a/services/seeker/src/torrents/providers/torrentgalaxy/search.ts b/services/seeker/src/torrents/providers/torrentgalaxy/search.ts deleted file mode 100644 index 5906d83..0000000 --- a/services/seeker/src/torrents/providers/torrentgalaxy/search.ts +++ /dev/null @@ -1,135 +0,0 @@ -import cheerio from "cheerio"; -import { request, convertToBytes } from "../../util"; - -const BASE_URL = "https://torrentgalaxy.mx/torrents.php?search="; - -export enum LanguageId { - AllLanguages = 0, - English = 1, - French = 2, - German = 3, - Italian = 4, - Japanese = 5, - Spanish = 6, - Russian = 7, - Hindi = 8, - OtherMultiple = 9, - Korean = 10, - Danish = 11, - Norwegian = 12, - Dutch = 13, - Chinese = 14, - Portuguese = 15, - Bengali = 16, - Polish = 17, - Turkish = 18, - Telugu = 19, - Urdu = 20, - Arabic = 21, - Swedish = 22, - Romanian = 23, - Thai = 24 -} - -export enum Language { - AllLanguages ="AllLanguages", - English ="English", - French ="French", - German ="German", - Italian ="Italian", - Japanese ="Japanese", - Spanish ="Spanish", - Russian ="Russian", - Hindi ="Hindi", - OtherMultiple ="OtherMultiple", - Korean ="Korean", - Danish ="Danish", - Norwegian ="Norwegian", - Dutch ="Dutch", - Chinese ="Chinese", - Portuguese ="Portuguese", - Bengali ="Bengali", - Polish ="Polish", - Turkish ="Turkish", - Telugu ="Telugu", - Urdu ="Urdu", - Arabic ="Arabic", - Swedish ="Swedish", - Romanian ="Romanian", - Thai ="Thai" -} - -export enum Category { - Documentaries = 9, - MoviesHD = 42, - MoviesSD = 1, - Movies4K = 3, - MoviesPacks = 4, - TVEpisodesHD = 41, - TVEPisodesSD = 5, - TVPacks = 6, - TVSports = 7 -} - -export enum Sort { - Date = "id", - Name = "name", - Size = "size", - Seeders = "seeders" -} - -export enum SortOrder { - Asc = "asc", - Desc = "desc", -} - -interface ITorrentGalaxyTorrent { - category: number, - language: Language, - name : string, - magnet : string, - size : number, - seeders : number, - leechers: number -} - -interface ITorrentGalaxyResults { - torrents: ITorrentGalaxyTorrent[], - total_results: number -} - -function scrapeRow($: cheerio.Root, row: cheerio.Cheerio): ITorrentGalaxyTorrent { - let children = row.children(); - let category = $(children[0]).find("a").attr("href")?.split("cat=")[1]; - let language = $(children[2]).find("img[title]").attr("title"); - let name = $(children[3]).text(); - let magnet = $(children[4]).find("a[href^='magnet']").first().attr("href"); - let [size, unit] = $(children[7]).text().split(" "); - let [seeders, leechers] = $(children[10]).text().slice(1, -1).split('/').map(v => parseInt(v)); - return { - category: parseInt(category), - size: convertToBytes(parseFloat(size), unit), - language, name, magnet, seeders, leechers - } -} - -function scrapeResults(response: string): ITorrentGalaxyResults { - let torrents: ITorrentGalaxyTorrent[] = []; - let $ = cheerio.load(response); - $(".tgxtable .tgxtablerow").each((_, elem) => { - torrents.push(scrapeRow($, $(elem))); - }); - return { - torrents, - total_results: parseInt($("#filterbox2 > span").text()) - }; -} - -/** - * Supports IMDb links too - */ -export async function search(query: string, language: LanguageId = LanguageId.AllLanguages, sort: Sort = Sort.Date, order: SortOrder = SortOrder.Desc) { - let res = await request(`${BASE_URL}${encodeURI(query)}&lang=${language}&sort=${sort}&order=${order}`); - let results = scrapeResults(res); - return results; -} diff --git a/services/seeker/src/torrents/ranking.ts b/services/seeker/src/torrents/ranking.ts deleted file mode 100644 index 2696e4f..0000000 --- a/services/seeker/src/torrents/ranking.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { Resolution } from "./parsing"; -import Torrent from "./Torrent"; - -/** - * Rank a list of torrents from best to worst to download - */ -export function rankTorrents(torrents: Torrent[]) { - torrents.sort(sortCompare); - let preferred = torrents.filter(selectPreferredTorrents); - return preferred; -} - -/** - * Filter out unwanted torrents - */ -function selectPreferredTorrents(torrent: Torrent) { - if (torrent.seeders == 0 || torrent.metadata.containsOtherLanguage) { - return false; - } - if (torrent.metadata.resolution == Resolution.HD4k) { - return torrent.size != null && torrent.size < 10*1024*1024*1024; // 10GB - } - return true; -} - -/** - * A comparator for ranking torrents - * - * @param a Left side - * @param b Right side - */ -function sortCompare(a: Torrent, b: Torrent) { - // Languages - let languageCmp = a.metadata.containsOtherLanguage - b.metadata.containsOtherLanguage; - if (languageCmp !== 0) { - return languageCmp; - } - // Resolution - let resolutionCmp = a.metadata.resolution - b.metadata.resolution; - if (resolutionCmp !== 0) { - return resolutionCmp; - } - - // If one has only a few seeds, don't worry about the other info. Prioritize seed count - if (a.seeders < 5 || b.seeders < 5) { - let seedersCmp = b.seeders - a.seeders; - if (seedersCmp != 0) { - return seedersCmp; - } - } - - // Sort by the file size - let fileSizeCmp = (a.size ?? 0) - (b.size ?? 0); - if (fileSizeCmp !== 0) { - return fileSizeCmp; - } - return 0; -} diff --git a/services/seeker/src/torrents/util.ts b/services/seeker/src/torrents/util.ts deleted file mode 100644 index cde8844..0000000 --- a/services/seeker/src/torrents/util.ts +++ /dev/null @@ -1,82 +0,0 @@ -import { parseString } from "xml2js"; -import https from "https"; - -/** - * Perform an RSS/XML request - */ -export function rssRequest(url: string) { - return new Promise((resolve, reject) => { - https.get(url, { headers: { "User-Agent": "Node", "Accept": "application/rss+xml" } }, (response) => { - if (response.statusCode !== 200) { - reject("Status error: " + response.statusCode); - return; - } - response.setEncoding("utf-8"); - let body = ""; - response.on("data", (chunk) => body += chunk); - response.on("end", () => parseString(body, (err, result) => { - if (err) { - reject(err); - return; - } - resolve(result); - })); - }); - }); -} - -/** - * Perform a generic GET request - */ - export function jsonRequest(url: string) { - return new Promise((resolve, reject) => { - https.get(url, { headers: { "User-Agent": "Node", "Accept": "*/*" } }, (response) => { - if (response.statusCode !== 200) { - reject("Status error: " + response.statusCode); - return; - } - response.setEncoding("utf-8"); - let body = ""; - response.on("data", (chunk) => body += chunk); - response.on("end", () => resolve(JSON.parse(body))); - }); - }); -} - -/** - * Perform a generic GET request - */ - export function request(url: string, timeout: number = 10000) { - return new Promise((resolve, reject) => { - https.get(url, { headers: { "User-Agent": "Node", "Accept": "*/*" }, timeout }, (response) => { - if (response.statusCode !== 200) { - reject("Status error: " + response.statusCode); - return; - } - response.setEncoding("utf-8"); - let body = ""; - response.on("data", (chunk) => body += chunk); - response.on("end", () => resolve(body)); - }).on("timeout", () => reject("timeout")); - }); -} - -export function sleep(ms: number) { - return new Promise(resolve => setTimeout(resolve, ms)); -} - -export function convertToBytes(size: number, unit: string, throwUnknownUnit: boolean = true) { - switch(unit.toUpperCase()) { - case "GB": - return Math.ceil(size*1024*1024*1024); - case "MB": - return Math.ceil(size*1024*1024); - case "KB": - return Math.ceil(size*1024*1024); - default: - if (throwUnknownUnit) { - throw new Error("Unknown unit provided"); - } - return Math.ceil(size); - } -}