From d8af1ae6c09557c0705061abb119824146b6dbbf Mon Sep 17 00:00:00 2001 From: flxxyz Date: Thu, 9 Nov 2023 12:10:28 +0800 Subject: [PATCH] feat: add optional httpsProxyAgent --- package.json | 2 + src/agent-phin.d.ts | 124 ++++++++++++++++++++++++++++++++++++++++++++ src/class/Eiyuu.ts | 28 +++++----- src/modifier.ts | 8 +-- 4 files changed, 147 insertions(+), 15 deletions(-) create mode 100644 src/agent-phin.d.ts diff --git a/package.json b/package.json index 1ba1aab..acd6126 100644 --- a/package.json +++ b/package.json @@ -55,7 +55,9 @@ }, "homepage": "https://eiyuu.js.org", "dependencies": { + "agent-phin": "^1.0.4", "cheerio": "^1.0.0-rc.12", + "https-proxy-agent": "^7.0.2", "phin": "^3.7.0" }, "devDependencies": { diff --git a/src/agent-phin.d.ts b/src/agent-phin.d.ts new file mode 100644 index 0000000..5b5ede2 --- /dev/null +++ b/src/agent-phin.d.ts @@ -0,0 +1,124 @@ +declare module "agent-phin"; + +import * as http from "http"; +import { URL } from "url"; +import { HttpsProxyAgent } from "https-proxy-agent"; + +interface IOptionsBase { + url: string | URL + method?: string + headers?: object + core?: http.ClientRequestArgs + followRedirects?: boolean + stream?: boolean + compression?: boolean + timeout?: number + hostname?: string + port?: number + path?: string + agent?: HttpsProxyAgent +} + +declare function phin(options: + phin.IJSONResponseOptions | + phin.IWithData | + phin.IWithForm): Promise> + +declare function phin(options: + phin.IStringResponseOptions | + phin.IWithData | + phin.IWithForm): Promise + +declare function phin(options: + phin.IStreamResponseOptions | + phin.IWithData | + phin.IWithForm): Promise + +declare function phin(options: + phin.IOptions | + phin.IWithData | + phin.IWithForm | + string): Promise + +declare namespace phin { + // Form and data property has been written this way so they're mutually exclusive. + export type IWithData = T & { + data: string | Buffer | object; + } + + export type IWithForm = T & { + form: { + [index: string]: string + } + } + + export interface IJSONResponseOptions extends IOptionsBase { + parse: "json" + } + + export interface IStringResponseOptions extends IOptionsBase { + parse: "string"; + } + + export interface IStreamResponseOptions extends IOptionsBase { + stream: true + } + + export interface IOptions extends IOptionsBase { + parse?: "none" + } + + export interface IJSONResponse extends http.IncomingMessage { + body: T + } + + export interface IStringResponse extends http.IncomingMessage { + body: string; + } + + export interface IStreamResponse extends http.IncomingMessage { + stream: http.IncomingMessage + } + + export interface IResponse extends http.IncomingMessage { + body: Buffer; + } + + // NOTE: Typescript cannot infer type of union callback on the consumer side + // https://github.com/Microsoft/TypeScript/pull/17819#issuecomment-363636904 + type IErrorCallback = (error: Error | string, response: null) => void + type ICallback = (error: null, response: NonNullable) => void + + export let promisified: typeof phin; + + export function unpromisified( + options: + IJSONResponseOptions | + IWithData | + IWithForm, + callback: IErrorCallback | ICallback>): void + + export function unpromisified( + options: + IStringResponseOptions | + IWithData | + IWithForm, + callback: IErrorCallback | ICallback): void + + export function unpromisified( + options: + IStreamResponseOptions | + IWithData | + IWithForm, + callback: IErrorCallback | ICallback): void + + export function unpromisified( + options: + IOptions | + IWithData | + IWithForm | + string, + callback: IErrorCallback | ICallback): void +} + +export = phin diff --git a/src/class/Eiyuu.ts b/src/class/Eiyuu.ts index e767de5..aacc75b 100644 --- a/src/class/Eiyuu.ts +++ b/src/class/Eiyuu.ts @@ -1,4 +1,5 @@ import { load } from "cheerio"; +import { HttpsProxyAgent } from "https-proxy-agent"; import { request, htmlDecode } from "../modifier"; import defaults from "../defaults"; import c from "../base"; @@ -21,15 +22,18 @@ export default class Eiyuu { private searchSortings: string; private searchSortingsMoebooruBased: string; private followRedirects: boolean; + private httpsProxyAgent?: HttpsProxyAgent; /** * Eiyuu * @param useragent custom useragent * @param followRedirects enable HTTP redirect following + * @param httpsProxyAgent custom https proxy agent */ - public constructor(useragent?: string, followRedirects?: boolean) { + public constructor(useragent?: string, followRedirects?: boolean, httpsProxyAgent?: HttpsProxyAgent) { this.useragent = useragent || defaults.useragent; this.followRedirects = followRedirects || defaults.followRedirects; + this.httpsProxyAgent = httpsProxyAgent; this.searchElements = "table.highlightable"; this.searchSortings = "*&sort=desc&order_by=index_count"; this.searchSortingsMoebooruBased = "*&type=&order=count"; @@ -62,7 +66,7 @@ export default class Eiyuu { try { const res = await request( this.danbooruURL, `*${query}`, "*%2A&search%5Border%5D=count", - this.useragent, this.followRedirects); + this.useragent, this.followRedirects, this.httpsProxyAgent); const $ = load(res.body); @@ -91,7 +95,7 @@ export default class Eiyuu { try { const res = await request( this.gelbooruURL, `*${query}`, this.searchSortings, - this.useragent, this.followRedirects); + this.useragent, this.followRedirects, this.httpsProxyAgent); const $ = load(res.body); @@ -118,7 +122,7 @@ export default class Eiyuu { try { const res = await request( this.hypnohubURL, `*${query}`, this.searchSortings, - this.useragent, this.followRedirects); + this.useragent, this.followRedirects, this.httpsProxyAgent); const $ = load(res.body); @@ -146,7 +150,7 @@ export default class Eiyuu { try { const res = await request( this.konachanURL, `*${query}`, this.searchSortingsMoebooruBased, - this.useragent, this.followRedirects); + this.useragent, this.followRedirects, this.httpsProxyAgent); const $ = load(res.body); @@ -174,7 +178,7 @@ export default class Eiyuu { try { const res = await request( this.lolibooruURL, `*${query}`, this.searchSortingsMoebooruBased, - this.useragent, this.followRedirects); + this.useragent, this.followRedirects, this.httpsProxyAgent); const $ = load(res.body); @@ -202,7 +206,7 @@ export default class Eiyuu { try { const res = await request( this.rule34URL, `*${query}`, this.searchSortings, - this.useragent, this.followRedirects); + this.useragent, this.followRedirects, this.httpsProxyAgent); const $ = load(res.body); @@ -229,7 +233,7 @@ export default class Eiyuu { try { const res = await request( this.realbooruURL, `*${query}`, this.searchSortings, - this.useragent, this.followRedirects); + this.useragent, this.followRedirects, this.httpsProxyAgent); const $ = load(res.body); @@ -256,7 +260,7 @@ export default class Eiyuu { try { const res = await request( this.safebooruURL, `*${query}`, this.searchSortings, - this.useragent, this.followRedirects); + this.useragent, this.followRedirects, this.httpsProxyAgent); const $ = load(res.body); @@ -283,7 +287,7 @@ export default class Eiyuu { try { const res = await request( this.tbibURL, `*${query}`, this.searchSortings, - this.useragent, this.followRedirects); + this.useragent, this.followRedirects, this.httpsProxyAgent); const $ = load(res.body); @@ -310,7 +314,7 @@ export default class Eiyuu { try { const res = await request( this.xbooruURL, `*${query}`, this.searchSortings, - this.useragent, this.followRedirects); + this.useragent, this.followRedirects, this.httpsProxyAgent); const $ = load(res.body); @@ -337,7 +341,7 @@ export default class Eiyuu { try { const res = await request( this.yandereURL, `*${query}`, this.searchSortingsMoebooruBased, - this.useragent, this.followRedirects); + this.useragent, this.followRedirects, this.httpsProxyAgent); const $ = load(res.body); diff --git a/src/modifier.ts b/src/modifier.ts index dfa2194..90a1524 100644 --- a/src/modifier.ts +++ b/src/modifier.ts @@ -1,4 +1,5 @@ -import p from "phin"; +import p from "agent-phin"; +import { HttpsProxyAgent } from "https-proxy-agent"; /** * Scrape request @@ -10,13 +11,14 @@ import p from "phin"; * @return {Promise} Phin response */ export async function request(url: string, query: string, sort: string, - useragent: string, redirect: boolean): Promise { + useragent: string, redirect: boolean, agent?: HttpsProxyAgent): Promise { const res = await p({ url: url + query + sort, "headers": { "User-Agent": useragent }, - followRedirects: redirect + followRedirects: redirect, + ...(agent ? { agent } : {}), }); return res; }