Improved the general torrent provider. Added RARBG as a provider

services/torrent-search/package.json

@@ -24,6 +24,7 @@
     "@autoplex-api/torrent-search": "^0.0.0",
     "@autoplex/ipc": "^0.0.0",
     "@autoplex/microservice": "^0.0.0",
+    "@autoplex/utils": "^0.0.0",
     "cheerio": "^1.0.0-rc.9",
     "node-ipc": "^9.1.4",
     "xml2js": "^0.4.23"

services/torrent-search/src/services/TorrentSearch.ts

@@ -36,12 +36,32 @@ export default class TorrentSearch extends InternalService
 		this.tvProviders = this.providers.filter(provider => provider.PROVIDES & MediaType.TvShows);
 	}
 
+	/**
+	 * Filter duplicate and blacklisted torrents
+	 */
+	protected filterTorrents(torrents: Torrent[], blacklist: string[]) {
+		let toRemove = new Set(blacklist);
+		return torrents.filter(torrent => {
+			let infoHash = torrent.infoHash();
+			if (infoHash === null) {
+				return true;
+			}
+			if (toRemove.has(infoHash)) {
+				return false;
+			}
+			toRemove.add(infoHash);
+			return true;
+		});
+	}
+
 	/**
 	 * Search for a movie
 	 */
 	public async searchMovie(movie: IMovieSearchInfo) {
 		let results = await Promise.all(this.movieProviders.map(provider => provider.searchMovie(movie)));
 		let torrents = (<Torrent[]>[]).concat(...results);
+
+		torrents = this.filterTorrents(torrents, movie.torrentBlacklist);
 		if (torrents.length == 0) {
 			return null;
 		}

services/torrent-search/src/torrents/Torrent.ts

@@ -3,6 +3,11 @@ import { ITorrentMetaInfo, parseMovieTorrentName } from "./parsing";
 
 export default class Torrent
 {
+	/**
+	 * The provider of the torrent
+	 */
+	public readonly provider: string;
+
 	/**
 	 * The name of the torrent
 	 */
@@ -34,13 +39,14 @@ export default class Torrent
 	 * @param name    The name of the torrent
 	 * @param size    The size of the torrent in bytes (if available)
 	 * @param seeders The number of seeders (if available)
-	 * @param link    The number of seeders (if available)
+	 * @param link    The link to the torrent (if available)
 	 */
-	public constructor(movie: IMovieSearchInfo, name: string, size?: number, seeders?: number, link?: string) {
-		this.name    = name.trim();
-		this.size    = size ?? null;
-		this.seeders = seeders ?? 1;
-		this.link    = link ?? null;
+	public constructor(provider: string, movie: IMovieSearchInfo, name: string, size?: number, seeders?: number, link?: string) {
+		this.provider = provider;
+		this.name     = name.trim();
+		this.size     = size ?? null;
+		this.seeders  = seeders ?? 1;
+		this.link     = link ?? null;
 		this.metadata = parseMovieTorrentName(name, movie);
 	}
 
@@ -54,6 +60,17 @@ export default class Torrent
 		return this.link;
 	}
 
+	/**
+	 * Get the info hash of the torrent if it exists
+	 */
+	public infoHash() {
+		if (this.link === null) {
+			return null;
+		}
+		let infoHash = (this.link.match(/(?<=^magnet:\?xt=urn:btih:)[0-9a-f]+/gi) ?? [])[0];
+		return infoHash ?? null;
+	}
+
 	/**
 	 * Serialize this torrent into a string
 	 */

services/torrent-search/src/torrents/providers/Provider.ts

@@ -1,4 +1,5 @@
 import { IMovieSearchInfo } from "@autoplex-api/torrent-search";
+import { sleep } from "@autoplex/utils";
 import Torrent from "../Torrent";
 
 /**
@@ -10,17 +11,63 @@ export enum MediaType {
 	TvShows = 0x2
 }
 
+/**
+ * The search method type
+ */
+type SearchMethod = (...args: any[]) => Promise<Torrent[]>;
+
+/**
+ * The base implementation for a torrent provider
+ */
 export default abstract class Provider
 {
+	/**
+	 * The name of the provider
+	 */
+	public abstract readonly NAME: string;
+
 	/**
 	 * Indicate what media types the provider supports
 	 */
 	public abstract readonly PROVIDES: MediaType;
 
 	/**
-	 * Search for movies
+	 * How much to throttle requests
+	 */
+	protected abstract readonly THROTTLE: number;
+
+	/**
+	 * Retry search requests if necessary
+	 */
+	protected readonly SEARCH_RETRIES: number = 0;
+
+	/**
+	 * Indicate if the API is currently authenticated
+	 */
+	protected async isAuthenticated() {
+		return true;
+	}
+
+	/**
+	 * Authenticate the API if necessary. Invoked once before each findMovieTorrents request
+	 */
+	protected async authenticate(): Promise<boolean> {
+		return true;
+	}
+
+	/**
+	 * Search for movies by the provided IMDb ID
 	 */
-	public abstract searchMovie(movie: IMovieSearchInfo): Promise<Torrent[]>;
+	protected async searchMovieImdb(imdbId: string, movie: IMovieSearchInfo): Promise<Torrent[]> {
+		return [];
+	}
+
+	/**
+	 * Search for movies by the title and year
+	 */
+	protected async searchMovieString(movie: IMovieSearchInfo): Promise<Torrent[]> {
+		return [];
+	}
 
 	// ---------------------------------------------------------------------------------------------
 
@@ -34,6 +81,15 @@ export default abstract class Provider
 	 */
 	protected searchQueue: (() => void)[] = [];
 
+	/**
+	 * Create a torrent object
+	 */
+	protected createTorrent(movie: IMovieSearchInfo, name: string, size?: number, seeders?: number,
+	                        link?: string)
+	{
+		return new Torrent(this.NAME, movie, name, size, seeders, link);
+	}
+
 	/**
 	 * Lock the search
 	 */
@@ -57,4 +113,62 @@ export default abstract class Provider
 			next();
 		}, delay);
 	}
+
+	/**
+	 * Await for authentication if necessary
+	 */
+	protected async awaitAuthentication() {
+		while (!await this.isAuthenticated()) {
+			try {
+				await this.authenticate();
+			} catch(e) {
+				console.error("Failed to authenticate", e);
+			}
+			await sleep(this.THROTTLE);
+		}
+	}
+
+	// Public Interface ----------------------------------------------------------------------------
+
+	/**
+	 * Search for content with a number of retries
+	 */
+	protected async search<M extends SearchMethod>(searchMethod: M, args: Parameters<M>,
+	                                               results: Torrent[])
+	{
+		for (let i = 0; i <= this.SEARCH_RETRIES; i++) {
+			try {
+				await this.awaitAuthentication();
+				let torrents = await searchMethod.apply(this, args);
+				return results.concat(torrents);
+			} catch(e) {
+				await sleep(this.THROTTLE);
+				continue;
+			}
+		}
+		return results;
+	}
+
+	/**
+	 * Search for movies
+	 */
+	public async searchMovie(movie: IMovieSearchInfo) {
+		// Accumulate a mutex lock
+		await this.lock();
+
+		// Create the results list
+		let results: Torrent[] = [];
+
+		// If an IMDb ID is available, search for movies by the IMDb ID
+		if (movie.imdbId !== null) {
+			results = await this.search(this.searchMovieImdb, [movie.imdbId, movie], results);
+		}
+
+		// Search by string (if supported)
+		results = await this.search(this.searchMovieString, [movie], results);
+
+		// Unlock and return the results
+		this.unlock(this.THROTTLE);
+		return results;
+	}
 }

services/torrent-search/src/torrents/providers/index.ts

@@ -1,10 +1,12 @@
 import Provider, { MediaType } from "./Provider";
+import Rarbg from "./rarbg";
 import TorrentGalaxy from "./torrentgalaxy";
 
 /**
  * Export all available torrent providers
  */
 export const providers = {
+	Rarbg,
 	TorrentGalaxy
 };
 

services/torrent-search/src/torrents/providers/rarbg/index.ts

@@ -0,0 +1,91 @@
+import { IMovieSearchInfo } from "@autoplex-api/torrent-search";
+import { jsonRequest } from "../../util";
+import Provider, { MediaType } from "../Provider";
+import { ISearchResult, ITokenResponse } from "./schema";
+
+/**
+ * The Base URL for the API
+ */
+const URL_BASE = "https://torrentapi.org/pubapi_v2.php?app_id=autoplex";
+
+/**
+ * How frequently to re-authenticate
+ */
+const AUTH_FREQUENCY = 1000*60*15; // 15 minutes
+
+export default class Rarbg extends Provider
+{
+	/**
+	 * The name of the provider
+	 */
+	public readonly NAME = "RARBG";
+
+	/**
+	 * Indicate the content that the provider provides
+	 */
+	public readonly PROVIDES = MediaType.Movies | MediaType.TvShows;
+
+	/**
+	 * Throttle requests;
+	 */
+	protected readonly THROTTLE = 1500;
+
+	/**
+	 * Since the API is sereverly broken, we need a bunch of retries
+	 */
+	protected readonly SEARCH_RETRIES = 30;
+
+	/**
+	 * Store the time when the API needs to re-authenticate
+	 */
+	protected nextAuthenticationTime: number = 0;
+
+	/**
+	 * The authentication token
+	 */
+	protected token!: string;
+
+	/**
+	 * Indicate if the API is currently authenticated
+	 */
+	protected async isAuthenticated() {
+		return Date.now() < this.nextAuthenticationTime;
+	}
+
+	/**
+	 * Authenticate the RARBG context
+	 */
+	protected async authenticate() {
+		let now = Date.now();
+		let [status, response] = await jsonRequest<ITokenResponse>(`${URL_BASE}&get_token=get_token`);
+		if (status !== 200) {
+			return false;
+		}
+		this.nextAuthenticationTime = now + AUTH_FREQUENCY;
+		this.token = response.token;
+		return true;
+	}
+
+	/**
+	 * Search for a movie
+	 */
+	public async searchMovieImdb(imdbId: string, movie: IMovieSearchInfo) {
+		let [_, response] = await jsonRequest<ISearchResult>(
+			`${URL_BASE}&token=${this.token}`
+			+ `&mode=search&search_imdb=${imdbId}`
+			+ "&format=json_extended&limit=100"
+		);
+		if (response.error_code !== undefined) {
+			if (response.error_code === 10) {
+				return []; // Successful search, no results found
+			}
+			throw new Error("No results found"); // API most likely broke, we need to retry.
+		}
+		if (response.torrent_results === undefined) {
+			throw new Error("Torrents should be defined here");
+		}
+		return response.torrent_results.map(result => this.createTorrent(
+			movie, result.title, result.size, result.seeders, result.download
+		));
+	}
+}

services/torrent-search/src/torrents/providers/rarbg/schema.ts

@@ -0,0 +1,52 @@
+export enum Category {
+	MOVIES_XVID        = 14,
+	MOVIES_XVID_720    = 48,
+	MOVIES_X264        = 17,
+	MOVIES_X264_1080   = 44,
+	MOVIES_X264_720    = 45,
+	MOVIES_X264_3D     = 47,
+	MOVIES_X264_4K     = 50,
+	MOVIES_X265_4K     = 51,
+	MOVIES_X265_4K_HDR = 52,
+	MOVIES_FULL_BD     = 42,
+	MOVIES_BD_REMUX    = 46,
+	TV_EPISODES        = 18,
+	TV_HD_EPISODES     = 41,
+	TV_UHD_EPISODES    = 49
+}
+
+export enum ErrorCode {
+	NoTokenSet     = 1,
+	InvalidToken   = 4,
+	InvalidImdb    = 9,
+	ImdbNotFound   = 10,
+	NoResultsFound = 20,
+	Error          = 22
+}
+
+export interface ITokenResponse {
+	token: string
+}
+
+export interface ISearchResult {
+	error          ?: string,
+	error_code     ?: number,
+	torrent_results?: IResult[]
+}
+
+export interface IResult {
+	title       : string,
+	category    : string,
+	download    : string,
+	seeders     : number,
+	leechers    : number,
+	size        : number,
+	pubdate     : string,
+	ranked      : number,
+	info_page   : string,
+	episode_info: {
+		imdb      : string|null,
+		tvdb      : string|null,
+		themoviedb: string|null,
+	}
+}

services/torrent-search/src/torrents/providers/torrentgalaxy/index.ts

@@ -1,36 +1,32 @@
 import { IMovieSearchInfo } from "@autoplex-api/torrent-search";
 import Provider, { MediaType } from "../Provider";
-import Torrent from "../../Torrent";
-import { search, Sort } from "./search";
-
-/**
- * Throttle the torrent search
- */
-const THROTTLE_SEARCH = 3000;
+import { search } from "./search";
+import { Sort } from "./schema";
 
 export default class TorrentGalaxy extends Provider
 {
+	/**
+	 * The torrent provider name
+	 */
+	public readonly NAME = "Torrent Galaxy";
+
 	/**
 	 * Indicate that this provider provides movies
 	 */
 	public readonly PROVIDES = MediaType.Movies;
 
+	/**
+	 * Throttle the requests
+	 */
+	protected readonly THROTTLE = 3000;
+
 	/**
 	 * Search for a movie
 	 */
-	public async searchMovie(movie: IMovieSearchInfo) {
-		if (movie.imdbId === null) {
-			return [];
-		}
-		await this.lock();
-		let torrents = await search(movie.imdbId, undefined, Sort.Seeders);
-		this.unlock(THROTTLE_SEARCH);
-		return torrents.torrents.map(torrent => new Torrent(
-			movie,
-			torrent.name,
-			torrent.size,
-			torrent.seeders,
-			torrent.magnet
+	public async searchMovieImdb(imdbId: string, movie: IMovieSearchInfo) {
+		let torrents = await search(imdbId, undefined, Sort.Seeders);
+		return torrents.torrents.map(torrent => this.createTorrent(
+			movie, torrent.name, torrent.size,torrent.seeders, torrent.magnet
 		));
 	}
 }

services/torrent-search/src/torrents/providers/torrentgalaxy/schema.ts

@@ -0,0 +1,89 @@
+export enum LanguageId {
+	AllLanguages  = 0,
+	English       = 1,
+	French        = 2,
+	German        = 3,
+	Italian       = 4,
+	Japanese      = 5,
+	Spanish       = 6,
+	Russian       = 7,
+	Hindi         = 8,
+	OtherMultiple = 9,
+	Korean        = 10,
+	Danish        = 11,
+	Norwegian     = 12,
+	Dutch         = 13,
+	Chinese       = 14,
+	Portuguese    = 15,
+	Bengali       = 16,
+	Polish        = 17,
+	Turkish       = 18,
+	Telugu        = 19,
+	Urdu          = 20,
+	Arabic        = 21,
+	Swedish       = 22,
+	Romanian      = 23,
+	Thai          = 24
+}
+
+export enum Language {
+	AllLanguages  ="AllLanguages",
+	English       ="English",
+	French        ="French",
+	German        ="German",
+	Italian       ="Italian",
+	Japanese      ="Japanese",
+	Spanish       ="Spanish",
+	Russian       ="Russian",
+	Hindi         ="Hindi",
+	OtherMultiple ="OtherMultiple",
+	Korean        ="Korean",
+	Danish        ="Danish",
+	Norwegian     ="Norwegian",
+	Dutch         ="Dutch",
+	Chinese       ="Chinese",
+	Portuguese    ="Portuguese",
+	Bengali       ="Bengali",
+	Polish        ="Polish",
+	Turkish       ="Turkish",
+	Telugu        ="Telugu",
+	Urdu          ="Urdu",
+	Arabic        ="Arabic",
+	Swedish       ="Swedish",
+	Romanian      ="Romanian",
+	Thai          ="Thai"
+}
+
+export enum Category {
+	Documentaries = 9,
+	MoviesHD      = 42,
+	MoviesSD      = 1,
+	Movies4K      = 3,
+	MoviesPacks   = 4,
+	TVEpisodesHD  = 41,
+	TVEPisodesSD  = 5,
+	TVPacks       = 6,
+	TVSports      = 7
+}
+
+export enum Sort {
+	Date   = "id",
+	Name    = "name",
+	Size    = "size",
+	Seeders = "seeders"
+}
+
+export enum SortOrder {
+	Asc  = "asc",
+	Desc = "desc",
+}
+
+export interface ITorrentGalaxyTorrent {
+	category: number,
+	language: Language,
+	name    : string,
+	magnet  : string,
+	size    : number,
+	seeders : number,
+	leechers: number
+}

services/torrent-search/src/torrents/providers/torrentgalaxy/search.ts

@@ -1,99 +1,10 @@
 import type { Cheerio, CheerioAPI, Element } from "cheerio";
 import cheerio from "cheerio";
+import { ITorrentGalaxyTorrent, Language, LanguageId, Sort, SortOrder} from "./schema";
 import { request, convertToBytes } from "../../util";
 
 const BASE_URL = "https://torrentgalaxy.mx/torrents.php?search=";
 
-export enum LanguageId {
-	AllLanguages  = 0,
-	English       = 1,
-	French        = 2,
-	German        = 3,
-	Italian       = 4,
-	Japanese      = 5,
-	Spanish       = 6,
-	Russian       = 7,
-	Hindi         = 8,
-	OtherMultiple = 9,
-	Korean        = 10,
-	Danish        = 11,
-	Norwegian     = 12,
-	Dutch         = 13,
-	Chinese       = 14,
-	Portuguese    = 15,
-	Bengali       = 16,
-	Polish        = 17,
-	Turkish       = 18,
-	Telugu        = 19,
-	Urdu          = 20,
-	Arabic        = 21,
-	Swedish       = 22,
-	Romanian      = 23,
-	Thai          = 24
-}
-
-export enum Language {
-	AllLanguages  ="AllLanguages",
-	English       ="English",
-	French        ="French",
-	German        ="German",
-	Italian       ="Italian",
-	Japanese      ="Japanese",
-	Spanish       ="Spanish",
-	Russian       ="Russian",
-	Hindi         ="Hindi",
-	OtherMultiple ="OtherMultiple",
-	Korean        ="Korean",
-	Danish        ="Danish",
-	Norwegian     ="Norwegian",
-	Dutch         ="Dutch",
-	Chinese       ="Chinese",
-	Portuguese    ="Portuguese",
-	Bengali       ="Bengali",
-	Polish        ="Polish",
-	Turkish       ="Turkish",
-	Telugu        ="Telugu",
-	Urdu          ="Urdu",
-	Arabic        ="Arabic",
-	Swedish       ="Swedish",
-	Romanian      ="Romanian",
-	Thai          ="Thai"
-}
-
-export enum Category {
-	Documentaries = 9,
-	MoviesHD      = 42,
-	MoviesSD      = 1,
-	Movies4K      = 3,
-	MoviesPacks   = 4,
-	TVEpisodesHD  = 41,
-	TVEPisodesSD  = 5,
-	TVPacks       = 6,
-	TVSports      = 7
-}
-
-export enum Sort {
-	Date   = "id",
-	Name    = "name",
-	Size    = "size",
-	Seeders = "seeders"
-}
-
-export enum SortOrder {
-	Asc  = "asc",
-	Desc = "desc",
-}
-
-interface ITorrentGalaxyTorrent {
-	category: number,
-	language: Language,
-	name    : string,
-	magnet  : string,
-	size    : number,
-	seeders : number,
-	leechers: number
-}
-
 interface ITorrentGalaxyResults {
 	torrents: ITorrentGalaxyTorrent[],
 	total_results: number
@@ -130,7 +41,10 @@ function scrapeResults(response: string): ITorrentGalaxyResults {
  * Supports IMDb links too
  */
 export async function search(query: string, language: LanguageId = LanguageId.AllLanguages, sort: Sort = Sort.Date, order: SortOrder = SortOrder.Desc) {
-	let res = await request(`${BASE_URL}${encodeURI(query)}&lang=${language}&sort=${sort}&order=${order}`);
+	let [status, res] = await request(`${BASE_URL}${encodeURI(query)}&lang=${language}&sort=${sort}&order=${order}`);
+	if (status !== 200) {
+		throw Error("Non-200 code returned");
+	}
 	let results = scrapeResults(res);
 	return results;
 }

services/torrent-search/src/torrents/util.ts

@@ -4,9 +4,9 @@ import https from "https";
 /**
  * Perform an RSS/XML request
  */
-export function rssRequest<T = any>(url: string) {
-	return new Promise<T>((resolve, reject) => {
-		https.get(url, { headers: { "User-Agent": "Node", "Accept": "application/rss+xml" } }, (response) => {
+export function rssRequest<T = any>(url: string, headers: {[key: string]: string} = {}) {
+	return new Promise<[number|undefined, T]>((resolve, reject) => {
+		https.get(url, { headers: { "User-Agent": "Autoplex", "Accept": "application/rss+xml" } }, (response) => {
 			if (response.statusCode !== 200) {
 				reject("Status error: " + response.statusCode);
 				return;
@@ -19,7 +19,7 @@ export function rssRequest<T = any>(url: string) {
 					reject(err);
 					return;
 				}
-				resolve(result);
+				resolve([response.statusCode, result]);
 			}));
 		});
 	});
@@ -28,17 +28,13 @@ export function rssRequest<T = any>(url: string) {
 /**
  * Perform a generic GET request
  */
- export function jsonRequest<T = any>(url: string) {
-	return new Promise<T>((resolve, reject) => {
-		https.get(url, { headers: { "User-Agent": "Node", "Accept": "*/*" } }, (response) => {
-			if (response.statusCode !== 200) {
-				reject("Status error: " + response.statusCode);
-				return;
-			}
+ export function jsonRequest<T = any>(url: string, headers: {[key: string]: string} = {}) {
+	return new Promise<[number|undefined, T]>((resolve, reject) => {
+		https.get(url, { headers: { "User-Agent": "Autoplex", "Accept": "*/*", ...headers} }, (response) => {
 			response.setEncoding("utf-8");
 			let body = "";
 			response.on("data", (chunk) => body += chunk);
-			response.on("end", () => resolve(JSON.parse(body)));
+			response.on("end", () => resolve([response.statusCode, JSON.parse(body)]));
 		});
 	});
 }
@@ -46,9 +42,9 @@ export function rssRequest<T = any>(url: string) {
 /**
  * Perform a generic GET request
  */
- export function request(url: string, timeout: number = 10000) {
-	return new Promise<string>((resolve, reject) => {
-		https.get(url, { headers: { "User-Agent": "Node", "Accept": "*/*" }, timeout }, (response) => {
+ export function request(url: string, timeout: number = 10000, headers: {[key: string]: string} = {}) {
+	return new Promise<[number|undefined, string]>((resolve, reject) => {
+		https.get(url, { headers: { "User-Agent": "Autoplex", "Accept": "*/*", ...headers }, timeout }, (response) => {
 			if (response.statusCode !== 200) {
 				reject("Status error: " + response.statusCode);
 				return;
@@ -56,15 +52,11 @@ export function rssRequest<T = any>(url: string) {
 			response.setEncoding("utf-8");
 			let body = "";
 			response.on("data", (chunk) => body += chunk);
-			response.on("end", () => resolve(body));
+			response.on("end", () => resolve([response.statusCode, body]));
 		}).on("timeout", () => reject("timeout"));
 	});
 }
 
-export function sleep(ms: number) {
-	return new Promise(resolve => setTimeout(resolve, ms));
-}
-
 export function convertToBytes(size: number, unit: string, throwUnknownUnit: boolean = true) {
 	switch(unit.toUpperCase()) {
 		case "GB":