diff --git a/services/torrent-search/package.json b/services/torrent-search/package.json index 9de9618..0cbc679 100644 --- a/services/torrent-search/package.json +++ b/services/torrent-search/package.json @@ -24,6 +24,7 @@ "@autoplex-api/torrent-search": "^0.0.0", "@autoplex/ipc": "^0.0.0", "@autoplex/microservice": "^0.0.0", + "@autoplex/utils": "^0.0.0", "cheerio": "^1.0.0-rc.9", "node-ipc": "^9.1.4", "xml2js": "^0.4.23" diff --git a/services/torrent-search/src/services/TorrentSearch.ts b/services/torrent-search/src/services/TorrentSearch.ts index aed81ca..8159e45 100644 --- a/services/torrent-search/src/services/TorrentSearch.ts +++ b/services/torrent-search/src/services/TorrentSearch.ts @@ -36,12 +36,32 @@ export default class TorrentSearch extends InternalService this.tvProviders = this.providers.filter(provider => provider.PROVIDES & MediaType.TvShows); } + /** + * Filter duplicate and blacklisted torrents + */ + protected filterTorrents(torrents: Torrent[], blacklist: string[]) { + let toRemove = new Set(blacklist); + return torrents.filter(torrent => { + let infoHash = torrent.infoHash(); + if (infoHash === null) { + return true; + } + if (toRemove.has(infoHash)) { + return false; + } + toRemove.add(infoHash); + return true; + }); + } + /** * Search for a movie */ public async searchMovie(movie: IMovieSearchInfo) { let results = await Promise.all(this.movieProviders.map(provider => provider.searchMovie(movie))); let torrents = ([]).concat(...results); + + torrents = this.filterTorrents(torrents, movie.torrentBlacklist); if (torrents.length == 0) { return null; } diff --git a/services/torrent-search/src/torrents/Torrent.ts b/services/torrent-search/src/torrents/Torrent.ts index e48d8bc..9b689d1 100644 --- a/services/torrent-search/src/torrents/Torrent.ts +++ b/services/torrent-search/src/torrents/Torrent.ts @@ -3,6 +3,11 @@ import { ITorrentMetaInfo, parseMovieTorrentName } from "./parsing"; export default class Torrent { + /** + * The provider of the torrent + */ + public readonly provider: string; + /** * The name of the torrent */ @@ -34,13 +39,14 @@ export default class Torrent * @param name The name of the torrent * @param size The size of the torrent in bytes (if available) * @param seeders The number of seeders (if available) - * @param link The number of seeders (if available) + * @param link The link to the torrent (if available) */ - public constructor(movie: IMovieSearchInfo, name: string, size?: number, seeders?: number, link?: string) { - this.name = name.trim(); - this.size = size ?? null; - this.seeders = seeders ?? 1; - this.link = link ?? null; + public constructor(provider: string, movie: IMovieSearchInfo, name: string, size?: number, seeders?: number, link?: string) { + this.provider = provider; + this.name = name.trim(); + this.size = size ?? null; + this.seeders = seeders ?? 1; + this.link = link ?? null; this.metadata = parseMovieTorrentName(name, movie); } @@ -54,6 +60,17 @@ export default class Torrent return this.link; } + /** + * Get the info hash of the torrent if it exists + */ + public infoHash() { + if (this.link === null) { + return null; + } + let infoHash = (this.link.match(/(?<=^magnet:\?xt=urn:btih:)[0-9a-f]+/gi) ?? [])[0]; + return infoHash ?? null; + } + /** * Serialize this torrent into a string */ diff --git a/services/torrent-search/src/torrents/providers/Provider.ts b/services/torrent-search/src/torrents/providers/Provider.ts index 0b79396..075ca1a 100644 --- a/services/torrent-search/src/torrents/providers/Provider.ts +++ b/services/torrent-search/src/torrents/providers/Provider.ts @@ -1,4 +1,5 @@ import { IMovieSearchInfo } from "@autoplex-api/torrent-search"; +import { sleep } from "@autoplex/utils"; import Torrent from "../Torrent"; /** @@ -10,17 +11,63 @@ export enum MediaType { TvShows = 0x2 } +/** + * The search method type + */ +type SearchMethod = (...args: any[]) => Promise; + +/** + * The base implementation for a torrent provider + */ export default abstract class Provider { + /** + * The name of the provider + */ + public abstract readonly NAME: string; + /** * Indicate what media types the provider supports */ public abstract readonly PROVIDES: MediaType; /** - * Search for movies + * How much to throttle requests + */ + protected abstract readonly THROTTLE: number; + + /** + * Retry search requests if necessary + */ + protected readonly SEARCH_RETRIES: number = 0; + + /** + * Indicate if the API is currently authenticated + */ + protected async isAuthenticated() { + return true; + } + + /** + * Authenticate the API if necessary. Invoked once before each findMovieTorrents request + */ + protected async authenticate(): Promise { + return true; + } + + /** + * Search for movies by the provided IMDb ID */ - public abstract searchMovie(movie: IMovieSearchInfo): Promise; + protected async searchMovieImdb(imdbId: string, movie: IMovieSearchInfo): Promise { + return []; + } + + /** + * Search for movies by the title and year + */ + protected async searchMovieString(movie: IMovieSearchInfo): Promise { + return []; + } // --------------------------------------------------------------------------------------------- @@ -34,6 +81,15 @@ export default abstract class Provider */ protected searchQueue: (() => void)[] = []; + /** + * Create a torrent object + */ + protected createTorrent(movie: IMovieSearchInfo, name: string, size?: number, seeders?: number, + link?: string) + { + return new Torrent(this.NAME, movie, name, size, seeders, link); + } + /** * Lock the search */ @@ -57,4 +113,62 @@ export default abstract class Provider next(); }, delay); } + + /** + * Await for authentication if necessary + */ + protected async awaitAuthentication() { + while (!await this.isAuthenticated()) { + try { + await this.authenticate(); + } catch(e) { + console.error("Failed to authenticate", e); + } + await sleep(this.THROTTLE); + } + } + + // Public Interface ---------------------------------------------------------------------------- + + /** + * Search for content with a number of retries + */ + protected async search(searchMethod: M, args: Parameters, + results: Torrent[]) + { + for (let i = 0; i <= this.SEARCH_RETRIES; i++) { + try { + await this.awaitAuthentication(); + let torrents = await searchMethod.apply(this, args); + return results.concat(torrents); + } catch(e) { + await sleep(this.THROTTLE); + continue; + } + } + return results; + } + + /** + * Search for movies + */ + public async searchMovie(movie: IMovieSearchInfo) { + // Accumulate a mutex lock + await this.lock(); + + // Create the results list + let results: Torrent[] = []; + + // If an IMDb ID is available, search for movies by the IMDb ID + if (movie.imdbId !== null) { + results = await this.search(this.searchMovieImdb, [movie.imdbId, movie], results); + } + + // Search by string (if supported) + results = await this.search(this.searchMovieString, [movie], results); + + // Unlock and return the results + this.unlock(this.THROTTLE); + return results; + } } diff --git a/services/torrent-search/src/torrents/providers/index.ts b/services/torrent-search/src/torrents/providers/index.ts index ed68576..a8c26be 100644 --- a/services/torrent-search/src/torrents/providers/index.ts +++ b/services/torrent-search/src/torrents/providers/index.ts @@ -1,10 +1,12 @@ import Provider, { MediaType } from "./Provider"; +import Rarbg from "./rarbg"; import TorrentGalaxy from "./torrentgalaxy"; /** * Export all available torrent providers */ export const providers = { + Rarbg, TorrentGalaxy }; diff --git a/services/torrent-search/src/torrents/providers/rarbg/index.ts b/services/torrent-search/src/torrents/providers/rarbg/index.ts new file mode 100644 index 0000000..65a3bf4 --- /dev/null +++ b/services/torrent-search/src/torrents/providers/rarbg/index.ts @@ -0,0 +1,91 @@ +import { IMovieSearchInfo } from "@autoplex-api/torrent-search"; +import { jsonRequest } from "../../util"; +import Provider, { MediaType } from "../Provider"; +import { ISearchResult, ITokenResponse } from "./schema"; + +/** + * The Base URL for the API + */ +const URL_BASE = "https://torrentapi.org/pubapi_v2.php?app_id=autoplex"; + +/** + * How frequently to re-authenticate + */ +const AUTH_FREQUENCY = 1000*60*15; // 15 minutes + +export default class Rarbg extends Provider +{ + /** + * The name of the provider + */ + public readonly NAME = "RARBG"; + + /** + * Indicate the content that the provider provides + */ + public readonly PROVIDES = MediaType.Movies | MediaType.TvShows; + + /** + * Throttle requests; + */ + protected readonly THROTTLE = 1500; + + /** + * Since the API is sereverly broken, we need a bunch of retries + */ + protected readonly SEARCH_RETRIES = 30; + + /** + * Store the time when the API needs to re-authenticate + */ + protected nextAuthenticationTime: number = 0; + + /** + * The authentication token + */ + protected token!: string; + + /** + * Indicate if the API is currently authenticated + */ + protected async isAuthenticated() { + return Date.now() < this.nextAuthenticationTime; + } + + /** + * Authenticate the RARBG context + */ + protected async authenticate() { + let now = Date.now(); + let [status, response] = await jsonRequest(`${URL_BASE}&get_token=get_token`); + if (status !== 200) { + return false; + } + this.nextAuthenticationTime = now + AUTH_FREQUENCY; + this.token = response.token; + return true; + } + + /** + * Search for a movie + */ + public async searchMovieImdb(imdbId: string, movie: IMovieSearchInfo) { + let [_, response] = await jsonRequest( + `${URL_BASE}&token=${this.token}` + + `&mode=search&search_imdb=${imdbId}` + + "&format=json_extended&limit=100" + ); + if (response.error_code !== undefined) { + if (response.error_code === 10) { + return []; // Successful search, no results found + } + throw new Error("No results found"); // API most likely broke, we need to retry. + } + if (response.torrent_results === undefined) { + throw new Error("Torrents should be defined here"); + } + return response.torrent_results.map(result => this.createTorrent( + movie, result.title, result.size, result.seeders, result.download + )); + } +} diff --git a/services/torrent-search/src/torrents/providers/rarbg/schema.ts b/services/torrent-search/src/torrents/providers/rarbg/schema.ts new file mode 100644 index 0000000..6ebbed7 --- /dev/null +++ b/services/torrent-search/src/torrents/providers/rarbg/schema.ts @@ -0,0 +1,52 @@ +export enum Category { + MOVIES_XVID = 14, + MOVIES_XVID_720 = 48, + MOVIES_X264 = 17, + MOVIES_X264_1080 = 44, + MOVIES_X264_720 = 45, + MOVIES_X264_3D = 47, + MOVIES_X264_4K = 50, + MOVIES_X265_4K = 51, + MOVIES_X265_4K_HDR = 52, + MOVIES_FULL_BD = 42, + MOVIES_BD_REMUX = 46, + TV_EPISODES = 18, + TV_HD_EPISODES = 41, + TV_UHD_EPISODES = 49 +} + +export enum ErrorCode { + NoTokenSet = 1, + InvalidToken = 4, + InvalidImdb = 9, + ImdbNotFound = 10, + NoResultsFound = 20, + Error = 22 +} + +export interface ITokenResponse { + token: string +} + +export interface ISearchResult { + error ?: string, + error_code ?: number, + torrent_results?: IResult[] +} + +export interface IResult { + title : string, + category : string, + download : string, + seeders : number, + leechers : number, + size : number, + pubdate : string, + ranked : number, + info_page : string, + episode_info: { + imdb : string|null, + tvdb : string|null, + themoviedb: string|null, + } +} diff --git a/services/torrent-search/src/torrents/providers/torrentgalaxy/index.ts b/services/torrent-search/src/torrents/providers/torrentgalaxy/index.ts index 7d7d81d..6dacab1 100644 --- a/services/torrent-search/src/torrents/providers/torrentgalaxy/index.ts +++ b/services/torrent-search/src/torrents/providers/torrentgalaxy/index.ts @@ -1,36 +1,32 @@ import { IMovieSearchInfo } from "@autoplex-api/torrent-search"; import Provider, { MediaType } from "../Provider"; -import Torrent from "../../Torrent"; -import { search, Sort } from "./search"; - -/** - * Throttle the torrent search - */ -const THROTTLE_SEARCH = 3000; +import { search } from "./search"; +import { Sort } from "./schema"; export default class TorrentGalaxy extends Provider { + /** + * The torrent provider name + */ + public readonly NAME = "Torrent Galaxy"; + /** * Indicate that this provider provides movies */ public readonly PROVIDES = MediaType.Movies; + /** + * Throttle the requests + */ + protected readonly THROTTLE = 3000; + /** * Search for a movie */ - public async searchMovie(movie: IMovieSearchInfo) { - if (movie.imdbId === null) { - return []; - } - await this.lock(); - let torrents = await search(movie.imdbId, undefined, Sort.Seeders); - this.unlock(THROTTLE_SEARCH); - return torrents.torrents.map(torrent => new Torrent( - movie, - torrent.name, - torrent.size, - torrent.seeders, - torrent.magnet + public async searchMovieImdb(imdbId: string, movie: IMovieSearchInfo) { + let torrents = await search(imdbId, undefined, Sort.Seeders); + return torrents.torrents.map(torrent => this.createTorrent( + movie, torrent.name, torrent.size,torrent.seeders, torrent.magnet )); } } diff --git a/services/torrent-search/src/torrents/providers/torrentgalaxy/schema.ts b/services/torrent-search/src/torrents/providers/torrentgalaxy/schema.ts new file mode 100644 index 0000000..563c15c --- /dev/null +++ b/services/torrent-search/src/torrents/providers/torrentgalaxy/schema.ts @@ -0,0 +1,89 @@ +export enum LanguageId { + AllLanguages = 0, + English = 1, + French = 2, + German = 3, + Italian = 4, + Japanese = 5, + Spanish = 6, + Russian = 7, + Hindi = 8, + OtherMultiple = 9, + Korean = 10, + Danish = 11, + Norwegian = 12, + Dutch = 13, + Chinese = 14, + Portuguese = 15, + Bengali = 16, + Polish = 17, + Turkish = 18, + Telugu = 19, + Urdu = 20, + Arabic = 21, + Swedish = 22, + Romanian = 23, + Thai = 24 +} + +export enum Language { + AllLanguages ="AllLanguages", + English ="English", + French ="French", + German ="German", + Italian ="Italian", + Japanese ="Japanese", + Spanish ="Spanish", + Russian ="Russian", + Hindi ="Hindi", + OtherMultiple ="OtherMultiple", + Korean ="Korean", + Danish ="Danish", + Norwegian ="Norwegian", + Dutch ="Dutch", + Chinese ="Chinese", + Portuguese ="Portuguese", + Bengali ="Bengali", + Polish ="Polish", + Turkish ="Turkish", + Telugu ="Telugu", + Urdu ="Urdu", + Arabic ="Arabic", + Swedish ="Swedish", + Romanian ="Romanian", + Thai ="Thai" +} + +export enum Category { + Documentaries = 9, + MoviesHD = 42, + MoviesSD = 1, + Movies4K = 3, + MoviesPacks = 4, + TVEpisodesHD = 41, + TVEPisodesSD = 5, + TVPacks = 6, + TVSports = 7 +} + +export enum Sort { + Date = "id", + Name = "name", + Size = "size", + Seeders = "seeders" +} + +export enum SortOrder { + Asc = "asc", + Desc = "desc", +} + +export interface ITorrentGalaxyTorrent { + category: number, + language: Language, + name : string, + magnet : string, + size : number, + seeders : number, + leechers: number +} diff --git a/services/torrent-search/src/torrents/providers/torrentgalaxy/search.ts b/services/torrent-search/src/torrents/providers/torrentgalaxy/search.ts index 64511bc..6cd1ba2 100644 --- a/services/torrent-search/src/torrents/providers/torrentgalaxy/search.ts +++ b/services/torrent-search/src/torrents/providers/torrentgalaxy/search.ts @@ -1,99 +1,10 @@ import type { Cheerio, CheerioAPI, Element } from "cheerio"; import cheerio from "cheerio"; +import { ITorrentGalaxyTorrent, Language, LanguageId, Sort, SortOrder} from "./schema"; import { request, convertToBytes } from "../../util"; const BASE_URL = "https://torrentgalaxy.mx/torrents.php?search="; -export enum LanguageId { - AllLanguages = 0, - English = 1, - French = 2, - German = 3, - Italian = 4, - Japanese = 5, - Spanish = 6, - Russian = 7, - Hindi = 8, - OtherMultiple = 9, - Korean = 10, - Danish = 11, - Norwegian = 12, - Dutch = 13, - Chinese = 14, - Portuguese = 15, - Bengali = 16, - Polish = 17, - Turkish = 18, - Telugu = 19, - Urdu = 20, - Arabic = 21, - Swedish = 22, - Romanian = 23, - Thai = 24 -} - -export enum Language { - AllLanguages ="AllLanguages", - English ="English", - French ="French", - German ="German", - Italian ="Italian", - Japanese ="Japanese", - Spanish ="Spanish", - Russian ="Russian", - Hindi ="Hindi", - OtherMultiple ="OtherMultiple", - Korean ="Korean", - Danish ="Danish", - Norwegian ="Norwegian", - Dutch ="Dutch", - Chinese ="Chinese", - Portuguese ="Portuguese", - Bengali ="Bengali", - Polish ="Polish", - Turkish ="Turkish", - Telugu ="Telugu", - Urdu ="Urdu", - Arabic ="Arabic", - Swedish ="Swedish", - Romanian ="Romanian", - Thai ="Thai" -} - -export enum Category { - Documentaries = 9, - MoviesHD = 42, - MoviesSD = 1, - Movies4K = 3, - MoviesPacks = 4, - TVEpisodesHD = 41, - TVEPisodesSD = 5, - TVPacks = 6, - TVSports = 7 -} - -export enum Sort { - Date = "id", - Name = "name", - Size = "size", - Seeders = "seeders" -} - -export enum SortOrder { - Asc = "asc", - Desc = "desc", -} - -interface ITorrentGalaxyTorrent { - category: number, - language: Language, - name : string, - magnet : string, - size : number, - seeders : number, - leechers: number -} - interface ITorrentGalaxyResults { torrents: ITorrentGalaxyTorrent[], total_results: number @@ -130,7 +41,10 @@ function scrapeResults(response: string): ITorrentGalaxyResults { * Supports IMDb links too */ export async function search(query: string, language: LanguageId = LanguageId.AllLanguages, sort: Sort = Sort.Date, order: SortOrder = SortOrder.Desc) { - let res = await request(`${BASE_URL}${encodeURI(query)}&lang=${language}&sort=${sort}&order=${order}`); + let [status, res] = await request(`${BASE_URL}${encodeURI(query)}&lang=${language}&sort=${sort}&order=${order}`); + if (status !== 200) { + throw Error("Non-200 code returned"); + } let results = scrapeResults(res); return results; } diff --git a/services/torrent-search/src/torrents/util.ts b/services/torrent-search/src/torrents/util.ts index cde8844..7c02e73 100644 --- a/services/torrent-search/src/torrents/util.ts +++ b/services/torrent-search/src/torrents/util.ts @@ -4,9 +4,9 @@ import https from "https"; /** * Perform an RSS/XML request */ -export function rssRequest(url: string) { - return new Promise((resolve, reject) => { - https.get(url, { headers: { "User-Agent": "Node", "Accept": "application/rss+xml" } }, (response) => { +export function rssRequest(url: string, headers: {[key: string]: string} = {}) { + return new Promise<[number|undefined, T]>((resolve, reject) => { + https.get(url, { headers: { "User-Agent": "Autoplex", "Accept": "application/rss+xml" } }, (response) => { if (response.statusCode !== 200) { reject("Status error: " + response.statusCode); return; @@ -19,7 +19,7 @@ export function rssRequest(url: string) { reject(err); return; } - resolve(result); + resolve([response.statusCode, result]); })); }); }); @@ -28,17 +28,13 @@ export function rssRequest(url: string) { /** * Perform a generic GET request */ - export function jsonRequest(url: string) { - return new Promise((resolve, reject) => { - https.get(url, { headers: { "User-Agent": "Node", "Accept": "*/*" } }, (response) => { - if (response.statusCode !== 200) { - reject("Status error: " + response.statusCode); - return; - } + export function jsonRequest(url: string, headers: {[key: string]: string} = {}) { + return new Promise<[number|undefined, T]>((resolve, reject) => { + https.get(url, { headers: { "User-Agent": "Autoplex", "Accept": "*/*", ...headers} }, (response) => { response.setEncoding("utf-8"); let body = ""; response.on("data", (chunk) => body += chunk); - response.on("end", () => resolve(JSON.parse(body))); + response.on("end", () => resolve([response.statusCode, JSON.parse(body)])); }); }); } @@ -46,9 +42,9 @@ export function rssRequest(url: string) { /** * Perform a generic GET request */ - export function request(url: string, timeout: number = 10000) { - return new Promise((resolve, reject) => { - https.get(url, { headers: { "User-Agent": "Node", "Accept": "*/*" }, timeout }, (response) => { + export function request(url: string, timeout: number = 10000, headers: {[key: string]: string} = {}) { + return new Promise<[number|undefined, string]>((resolve, reject) => { + https.get(url, { headers: { "User-Agent": "Autoplex", "Accept": "*/*", ...headers }, timeout }, (response) => { if (response.statusCode !== 200) { reject("Status error: " + response.statusCode); return; @@ -56,15 +52,11 @@ export function rssRequest(url: string) { response.setEncoding("utf-8"); let body = ""; response.on("data", (chunk) => body += chunk); - response.on("end", () => resolve(body)); + response.on("end", () => resolve([response.statusCode, body])); }).on("timeout", () => reject("timeout")); }); } -export function sleep(ms: number) { - return new Promise(resolve => setTimeout(resolve, ms)); -} - export function convertToBytes(size: number, unit: string, throwUnknownUnit: boolean = true) { switch(unit.toUpperCase()) { case "GB":