feat: add optional httpsProxyAgent

package.json

@@ -55,7 +55,9 @@
   },
   "homepage": "https://eiyuu.js.org",
   "dependencies": {
+    "agent-phin": "^1.0.4",
     "cheerio": "^1.0.0-rc.12",
+    "https-proxy-agent": "^7.0.2",
     "phin": "^3.7.0"
   },
   "devDependencies": {

src/agent-phin.d.ts

@@ -0,0 +1,124 @@
+declare module "agent-phin";
+
+import * as http from "http";
+import { URL } from "url";
+import { HttpsProxyAgent } from "https-proxy-agent";
+
+interface IOptionsBase {
+  url: string | URL
+  method?: string
+  headers?: object
+  core?: http.ClientRequestArgs
+  followRedirects?: boolean
+  stream?: boolean
+  compression?: boolean
+  timeout?: number
+  hostname?: string
+  port?: number
+  path?: string
+  agent?: HttpsProxyAgent<string>
+}
+
+declare function phin<T>(options:
+  phin.IJSONResponseOptions |
+  phin.IWithData<phin.IJSONResponseOptions> |
+  phin.IWithForm<phin.IJSONResponseOptions>): Promise<phin.IJSONResponse<T>>
+
+declare function phin(options:
+  phin.IStringResponseOptions |
+  phin.IWithData<phin.IStringResponseOptions> |
+  phin.IWithForm<phin.IStringResponseOptions>): Promise<phin.IStringResponse>
+
+declare function phin(options:
+  phin.IStreamResponseOptions |
+  phin.IWithData<phin.IStreamResponseOptions> |
+  phin.IWithForm<phin.IStreamResponseOptions>): Promise<phin.IStreamResponse>
+
+declare function phin(options:
+  phin.IOptions |
+  phin.IWithData<phin.IOptions> |
+  phin.IWithForm<phin.IOptions> |
+  string): Promise<phin.IResponse>
+
+declare namespace phin {
+  // Form and data property has been written this way so they're mutually exclusive.
+  export type IWithData<T extends IOptionsBase> = T & {
+    data: string | Buffer | object;
+  }
+
+  export type IWithForm<T extends IOptionsBase> = T & {
+    form: {
+      [index: string]: string
+    }
+  }
+
+  export interface IJSONResponseOptions extends IOptionsBase {
+    parse: "json"
+  }
+
+  export interface IStringResponseOptions extends IOptionsBase {
+    parse: "string";
+  }
+
+  export interface IStreamResponseOptions extends IOptionsBase {
+    stream: true
+  }
+
+  export interface IOptions extends IOptionsBase {
+    parse?: "none"
+  }
+
+  export interface IJSONResponse<T> extends http.IncomingMessage {
+    body: T
+  }
+
+  export interface IStringResponse extends http.IncomingMessage {
+    body: string;
+  }
+
+  export interface IStreamResponse extends http.IncomingMessage {
+    stream: http.IncomingMessage
+  }
+
+  export interface IResponse extends http.IncomingMessage {
+    body: Buffer;
+  }
+
+  // NOTE: Typescript cannot infer type of union callback on the consumer side
+  // https://github.com/Microsoft/TypeScript/pull/17819#issuecomment-363636904
+  type IErrorCallback = (error: Error | string, response: null) => void
+  type ICallback<T> = (error: null, response: NonNullable<T>) => void
+
+  export let promisified: typeof phin;
+
+  export function unpromisified<T>(
+    options:
+      IJSONResponseOptions |
+      IWithData<IJSONResponseOptions> |
+      IWithForm<IJSONResponseOptions>,
+    callback: IErrorCallback | ICallback<IJSONResponse<T>>): void
+
+  export function unpromisified(
+    options:
+      IStringResponseOptions |
+      IWithData<IStringResponseOptions> |
+      IWithForm<IStringResponseOptions>,
+    callback: IErrorCallback | ICallback<IStringResponse>): void
+
+  export function unpromisified(
+    options:
+      IStreamResponseOptions |
+      IWithData<IStreamResponseOptions> |
+      IWithForm<IStreamResponseOptions>,
+    callback: IErrorCallback | ICallback<IStreamResponse>): void
+
+  export function unpromisified(
+    options:
+      IOptions |
+      IWithData<IOptions> |
+      IWithForm<IOptions> |
+      string,
+    callback: IErrorCallback | ICallback<IResponse>): void
+}
+
+export = phin

src/class/Eiyuu.ts

@@ -1,4 +1,5 @@
 import { load } from "cheerio";
+import { HttpsProxyAgent } from "https-proxy-agent";
 import { request, htmlDecode } from "../modifier";
 import defaults from "../defaults";
 import c from "../base";
@@ -21,15 +22,18 @@ export default class Eiyuu {
   private searchSortings: string;
   private searchSortingsMoebooruBased: string;
   private followRedirects: boolean;
+  private httpsProxyAgent?: HttpsProxyAgent<string>;
 
   /**
    * Eiyuu
    * @param useragent custom useragent
    * @param followRedirects enable HTTP redirect following
+   * @param httpsProxyAgent custom https proxy agent
   */
-  public constructor(useragent?: string, followRedirects?: boolean) {
+  public constructor(useragent?: string, followRedirects?: boolean, httpsProxyAgent?: HttpsProxyAgent<string>) {
     this.useragent = useragent || defaults.useragent;
     this.followRedirects = followRedirects || defaults.followRedirects;
+    this.httpsProxyAgent = httpsProxyAgent;
     this.searchElements = "table.highlightable";
     this.searchSortings = "*&sort=desc&order_by=index_count";
     this.searchSortingsMoebooruBased = "*&type=&order=count";
@@ -62,7 +66,7 @@ export default class Eiyuu {
     try {
       const res = await request(
         this.danbooruURL, `*${query}`, "*%2A&search%5Border%5D=count",
-        this.useragent, this.followRedirects);
+        this.useragent, this.followRedirects, this.httpsProxyAgent);
 
       const $ = load(res.body);
 
@@ -91,7 +95,7 @@ export default class Eiyuu {
     try {
       const res = await request(
         this.gelbooruURL, `*${query}`, this.searchSortings,
-        this.useragent, this.followRedirects);
+        this.useragent, this.followRedirects, this.httpsProxyAgent);
 
       const $ = load(res.body);
 
@@ -118,7 +122,7 @@ export default class Eiyuu {
     try {
       const res = await request(
         this.hypnohubURL, `*${query}`, this.searchSortings,
-        this.useragent, this.followRedirects);
+        this.useragent, this.followRedirects, this.httpsProxyAgent);
 
       const $ = load(res.body);
 
@@ -146,7 +150,7 @@ export default class Eiyuu {
     try {
       const res = await request(
         this.konachanURL, `*${query}`, this.searchSortingsMoebooruBased,
-        this.useragent, this.followRedirects);
+        this.useragent, this.followRedirects, this.httpsProxyAgent);
 
       const $ = load(res.body);
 
@@ -174,7 +178,7 @@ export default class Eiyuu {
     try {
       const res = await request(
         this.lolibooruURL, `*${query}`, this.searchSortingsMoebooruBased,
-        this.useragent, this.followRedirects);
+        this.useragent, this.followRedirects, this.httpsProxyAgent);
 
       const $ = load(res.body);
 
@@ -202,7 +206,7 @@ export default class Eiyuu {
     try {
       const res = await request(
         this.rule34URL, `*${query}`, this.searchSortings,
-        this.useragent, this.followRedirects);
+        this.useragent, this.followRedirects, this.httpsProxyAgent);
 
       const $ = load(res.body);
 
@@ -229,7 +233,7 @@ export default class Eiyuu {
     try {
       const res = await request(
         this.realbooruURL, `*${query}`, this.searchSortings,
-        this.useragent, this.followRedirects);
+        this.useragent, this.followRedirects, this.httpsProxyAgent);
 
       const $ = load(res.body);
 
@@ -256,7 +260,7 @@ export default class Eiyuu {
     try {
       const res = await request(
         this.safebooruURL, `*${query}`, this.searchSortings,
-        this.useragent, this.followRedirects);
+        this.useragent, this.followRedirects, this.httpsProxyAgent);
 
       const $ = load(res.body);
 
@@ -283,7 +287,7 @@ export default class Eiyuu {
     try {
       const res = await request(
         this.tbibURL, `*${query}`, this.searchSortings,
-        this.useragent, this.followRedirects);
+        this.useragent, this.followRedirects, this.httpsProxyAgent);
 
       const $ = load(res.body);
 
@@ -310,7 +314,7 @@ export default class Eiyuu {
     try {
       const res = await request(
         this.xbooruURL, `*${query}`, this.searchSortings,
-        this.useragent, this.followRedirects);
+        this.useragent, this.followRedirects, this.httpsProxyAgent);
 
       const $ = load(res.body);
 
@@ -337,7 +341,7 @@ export default class Eiyuu {
     try {
       const res = await request(
         this.yandereURL, `*${query}`, this.searchSortingsMoebooruBased,
-        this.useragent, this.followRedirects);
+        this.useragent, this.followRedirects, this.httpsProxyAgent);
 
       const $ = load(res.body);
 

src/modifier.ts

@@ -1,4 +1,5 @@
-import p from "phin";
+import p from "agent-phin";
+import { HttpsProxyAgent } from "https-proxy-agent";
 
 /**
  * Scrape request
@@ -10,13 +11,14 @@ import p from "phin";
  * @return {Promise<p.IResponse>} Phin response
  */
 export async function request(url: string, query: string, sort: string,
-  useragent: string, redirect: boolean): Promise<p.IResponse> {
+  useragent: string, redirect: boolean, agent?: HttpsProxyAgent<string>): Promise<p.IResponse> {
   const res = await p({
     url: url + query + sort,
     "headers": {
       "User-Agent": useragent
     },
-    followRedirects: redirect
+    followRedirects: redirect,
+    ...(agent ? { agent } : {}),
   });
   return res;
 }