Contributing to open source project JS-DOM

"use strict";
const request = require("request");

module.exports = cookieJar => {
  const jarWrapper = request.jar();
  jarWrapper._jar = cookieJar;
  return jarWrapper;
};

// ...
const request = require("request");
// ...
const wrapCookieJarForRequest = require("../helpers/wrap-cookie-jar-for-request");
// ...
  function doRequest() {
    try {
      const client = request(options);

      if (hasBody && flag.formData) {
        const form = client.form();
        for (const entry of body) {
          form.append(entry.name, entry.value, entry.options);
        }
      }

      return client;
    } catch (e) {
      const client = new EventEmitter();
      process.nextTick(() => client.emit("error", e));
      return client;
    }
  }
/// ...

// ...
const request = require("request");
// ...
/**
 * Reads a static fixture file as utf8.
 * If running tests from node, the file will be read from the file system
 * If running tests using karma, a http request will be performed to retrieve the file using karma's server.
 * @param {string} relativePath Relative path within the test directory. For example "jsdom/files/test.html"
 */
exports.readTestFixture = relativePath => {
  const useRequest = exports.inBrowserContext();

  return exports.nodeResolverPromise(nodeResolver => {
    if (useRequest) {
      request.get(exports.getTestFixtureUrl(relativePath), { timeout: 5000 }, nodeResolver);
    } else {
      fs.readFile(path.resolve(__dirname, relativePath), { encoding: "utf8" }, nodeResolver);
    }
  })
  // request passes (error, response, content) to the callback
  // we are only interested in the `content`
    .then(result => useRequest ? result[1] : result);
};

// ...
const request = require("request-promise-native");
const wrapCookieJarForRequest = require("../../living/helpers/wrap-cookie-jar-for-request");
// ...
  fetch(urlString, options = {}) {
    const url = parseURL(urlString);
    // ...
    switch (url.scheme) {
      // ...
      case "http":
      case "https": {
        const requestOptions = this._getRequestOptions(options);
        return request(urlString, requestOptions);
      }
      // ...
    }
  }

// ...
const requestHead = require("request-promise-native").head;
// ...
function pollForServer(url) {
  return requestHead(url, { strictSSL: false })
    .then(() => {
  // ...
}

// ...
const fetch = require("node-fetch");
// ...
exports.readTestFixture = relativePath => {
  const useRequest = exports.inBrowserContext();

  if (useRequest) {
    const url = exports.getTestFixtureUrl(relativePath);
    // timeout is a node-fetch specific extention.
    fetch(url, { timeout: 5000 }).then((response) => {
      if (!response.ok) {
        throw new Error(`Unexpected status ${response.status} fetching ${url}`);
      }
      return response.text();
    });
  } else {
    return exports.nodeResolverPromise(nodeResolver => {
      fs.readFile(path.resolve(__dirname, relativePath), { encoding: "utf8" }, nodeResolver);
    });
  }
};

exports.readTestFixture = relativePath => {
  const useRequest = exports.inBrowserContext();
  if (useRequest) {
    throw new Error("???");
// ...

/**
 * Reads a static fixture file as utf8.
 * If running tests from node, the file will be read from the file system
 * If running tests using karma, a http request will be performed to retrieve the file using karma's server.
 * @param {string} relativePath Relative path within the test directory. For example "jsdom/files/test.html"
 */
exports.readTestFixture = relativePath => {

  if (useRequest) {
    const url = exports.getTestFixtureUrl(relativePath);
    // timeout is a node-fetch specific extension
    return fetch(url, { timeout: 5000 }).then((response) => {
      if (!response.ok) {
        throw new Error(`Unexpected status ${response.status} fetching ${url}`);
      }
      return response.text();
    });
  }

  if (exports.inBrowserContext()) {
    return fetch(exports.getTestFixtureUrl(relativePath)).then((response) => {
      if (!response.ok) {
        throw new Error(`Unexpected status ${response.status} fetching ${response.location}`);
      }
      return response.text();
    });
  }

// ...
const requestHead = require("request-promise-native").head;
// ...
function pollForServer(url) {
  return requestHead(url, { strictSSL: false })
    .then(() => {
      console.log(`WPT server at ${url} is up!`);
      return url;
    })
    .catch(err => {
      console.log(`WPT server at ${url} is not up yet (${err.message}); trying again`);
      return new Promise(resolve => {
        setTimeout(() => resolve(pollForServer(url)), 500);
      });
    });
}

const fetch = require("node-fetch");
const https = require("https");
// ...

const httpsAgent = new https.Agent({
  rejectUnauthorized: false,
});

function pollForServer(url) {
  const agent = url.startsWith("https")
    ? new https.Agent({ rejectUnauthorized: false })
    : null;
  return fetch(url, { method: "HEAD", agent })
    .then(({ ok, status }) => {
      if (!ok) {
        throw new Error(`Unexpected status=${status}`);
      }
      console.log(`WPT server at ${url} is up!`);
      return url;
    })
    .catch(err => {
      console.log(`WPT server at ${url} is not up yet (${err.message}); trying again`);
      return new Promise(resolve => {
        setTimeout(() => resolve(pollForServer(url)), 500);
      });
    });
}

"use strict";
const request = require("request");

module.exports = cookieJar => {
  const jarWrapper = request.jar();
  jarWrapper._jar = cookieJar;
  return jarWrapper;
};

// ...
const request = require("request-promise-native");
const wrapCookieJarForRequest = require("../../living/helpers/wrap-cookie-jar-for-request");
// ...
  _getRequestOptions({ cookieJar, referrer, accept = "*/*" }) {
    const requestOptions = {
      encoding: null,
      gzip: true,
      jar: wrapCookieJarForRequest(cookieJar),
      strictSSL: this._strictSSL,
      proxy: this._proxy,
      forever: true,
      headers: {
        "User-Agent": this._userAgent,
        "Accept-Language": "en",
        Accept: accept
      }
    };

    if (referrer && !IS_BROWSER) {
      requestOptions.headers.referer = referrer;
    }

    return requestOptions;
  }
// ...
  fetch(urlString, options = {}) {
    const url = parseURL(urlString);
    // ...
    switch (url.scheme) {
      // ...
      case "http":
      case "https": {
        const requestOptions = this._getRequestOptions(options);
        return request(urlString, requestOptions);
      }
      // ...
    }
  }

_getFetchOptions({ cookieJar, referrer, accept = "*/*" }) {
  /** @type RequestInit */
  const fetchOptions = {};

  // I don't know what these variables hold exactly - let's log them!
  console.log("cookieJar", cookieJar);
  console.log("this._proxy", this._proxy);

  fetchOptions.headers = {
    "User-Agent": this._userAgent,
    "Accept-Language": "en",
    Accept: accept,
  };

  if (!IS_BROWSER) {
    const httpAgent = new http.Agent({ keepAlive: true });
    const httpsAgent = new https.Agent({ keepAlive: true, rejectUnauthorized: this._strictSSL });

    fetchOptions.headers.referrer = referrer;
    fetchOptions.agent = (url) => url.protocol == 'http:' ? httpAgent : httpsAgent;
  }

  return fetchOptions;
}

// ...
case "http":
case "https": {
  return fetch(urlString, this._getFetchOptions(options))
    .then((response) => {
      if (!response.ok) {
        throw new Error(`Unexpected status=${response.status} for ${urlString}`);
      }
      return response.arrayBuffer();
    })
}

_getFetchOptions({ cookieJar, referrer, accept = "*/*" }) {
  /** @type RequestInit */
  const fetchOptions = {};

  // I don't know what these variables hold exactly - let's log them!
  console.log("cookieJar", cookieJar);
  console.log("this._proxy", this._proxy);

  fetchOptions.headers = {
    "User-Agent": this._userAgent,
    "Accept-Language": "en",
    "Accept-Encoding": "gzip",
    Accept: accept,
  };

  if (!IS_BROWSER) {
    const httpAgent = new http.Agent({ keepAlive: true });
    const httpsAgent = new https.Agent({ keepAlive: true, rejectUnauthorized: this._strictSSL });

    fetchOptions.headers.referrer = referrer;
    fetchOptions.agent = (url) => url.protocol == 'http:' ? httpAgent : httpsAgent;
  }

  return fetchOptions;
}

// ...
case "http":
case "https": {
  const cookieJar = options.cookieJar;
  cookieJar.__setCookie = cookieJar.setCookie;
  cookieJar.setCookie = (...args) => {
    if (args.length === 3) {
      args.splice(2, 0, {});
    }
    if (args.length === 4) {
      args[2].ignoreError = true;
    }
    return cookieJar.__setCookie(...args);
  }
  const targetFetch = fetchCookie(fetch, cookieJar);
  const fetchOptions = this._getFetchOptions(options);
  return targetFetch(urlString, fetchOptions)
    .then((response) => {
      if (!response.ok) {
        throw new Error(`Unexpected status=${response.status} for ${urlString}`);
      }
      return response.arrayBuffer();
    });
}

const req = resourceLoaderForInitialRequest.fetch(url, {
  accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
  cookieJar: options.cookieJar,
  referrer: options.referrer
});

return req.then(body => {
  const res = req.response;

  options = Object.assign(options, {
    url: req.href + originalHash,
    contentType: res.headers["content-type"],
    referrer: req.getHeader("referer")
  });

  return new JSDOM(body, options);
});

const cookieJar = options.cookieJar;
cookieJar.__setCookie = cookieJar.setCookie;
cookieJar.setCookie = (...args) => { /* ... */ }
const targetFetch = fetchCookie(fetch, cookieJar);
const fetchOptions = this._getFetchOptions(options);
const fetchResult = targetFetch(urlString, fetchOptions);

let result;
result = {
  response: null,
  href: urlString,
  then: fetchResult.then((response) => {
    if (!response.ok) {
      throw new Error(`Unexpected status=${response.status} for ${urlString}`);
    }
    result.response = response;
    return response.arrayBuffer();
  }).then.bind(fetchResult),
  catch: fetchResult.catch.bind(fetchResult),
  getHeader(name) {
    return fetchOptions.headers[name];
  }
};

return result;

it("should share cookies when a cookie jar is shared", () => {
  const cookieJar = new CookieJar();

  return JSDOM.fromURL(testHost + "/TestPath/set-cookie-from-server", { cookieJar }).then(() => {
    return JSDOM.fromURL(testHost + "/TestPath/html-get-cookie-header", { cookieJar });
  }).then(({ window }) => {
    const cookieHeader = window.document.querySelector(".cookie-header").innerHTML;

    assertCookies(cookieHeader, [
      "Test1=Basic",
      "Test2=PathMatch",
      "Test6=HttpOnly",
      "Test9=Duplicate",
      "Test10={\"prop1\":5,\"prop2\":\"value\"}",
      "Malformed"
    ]);

    assertCookies(window.document.cookie, [
      "Test1=Basic",
      "Test2=PathMatch",
      "Test9=Duplicate",
      "Test10={\"prop1\":5,\"prop2\":\"value\"}",
      "Malformed"
    ]);
  });
});

{ html: '[object Response]' }

const fetchOptions = this._getFetchOptions(options);
const fetchPromise = targetFetch(urlString, fetchOptions);

let result;
const then = function(onfulfilled, onrejected) {
  return fetchPromise.then((response) => {
    if (!response.ok) {
      throw new Error(`Unexpected status=${response.status} for ${urlString}`);
    }
    result.response = response;
    return response.arrayBuffer();
  }).then(onfulfilled, onrejected);
};

result = {
  response: null,
  href: urlString,
  then,
  catch: function(onrejected) { return then(undefined, onrejected) },
  getHeader(name) {
    return fetchOptions.headers[name];
  }
};

return result;

const then = function(onfulfilled, onrejected) {
  return fetchPromise.then((response) => {
    if (!response.ok) {
      throw new Error(`Unexpected status=${response.status} for ${urlString}`);
    }
    result.response = response;
    return response.arrayBuffer();
  })
  .then((arrayBuffer) => {
    if (typeof Buffer === "undefined") {
      return arrayBuffer;
    } else {
      return Buffer.from(arrayBuffer);
    }
  })
  .then(onfulfilled, onrejected);
};

/**
 * Manage all the request and it is able to abort
 * all pending request.
 */
module.exports = class RequestManager {
  // ...
  close() {
    for (const openedRequest of this.openedRequests) {
      openedRequest.abort();
    }
    this.openedRequests = [];
  }
  // ...
};

result = {
  response: null,
  abort: () => { console.log("TODO ABORT"); },
  href: urlString,
  then,
  catch: function(onrejected) { return then(undefined, onrejected) },
  getHeader(name) {
    return fetchOptions.headers[name];
  }
};

const AbortController = require("abort-controller");

// ...
const targetFetch = fetchCookie(fetch, cookieJar);
const fetchOptions = this._getFetchOptions(options);
const abortController = new AbortController();
fetchOptions.signal = abortController.signal;
const fetchPromise = targetFetch(urlString, fetchOptions);

let result;
const then = function(onfulfilled, onrejected) {
  return fetchPromise.then((response) => {
    if (!response.ok) {
      throw new Error(`Unexpected status=${response.status} for ${urlString}`);
    }
    result.response = response;
    return response.arrayBuffer();
  })
  .then((arrayBuffer) => typeof Buffer === "undefined" ? arrayBuffer : Buffer.from(arrayBuffer))
  .then(onfulfilled, onrejected);
};

result = {
  response: null,
  abort: () => { abortController.abort(); },
  href: urlString,
  then,
  catch: function(onrejected) { return then(undefined, onrejected) },
  getHeader(name) {
    return fetchOptions.headers[name];
  }
};

let aborted = false;
let result;
const then = function(onfulfilled, onrejected) {
  return fetchPromise.then((response) => {
    if (!response.ok) {
      throw new Error(`Unexpected status=${response.status} for ${urlString}`);
    }
    result.response = response;
    return response.arrayBuffer();
  })
  .then((arrayBuffer) => typeof Buffer === "undefined" ? arrayBuffer : Buffer.from(arrayBuffer))
  .then((result) => { if (!aborted) return onfulfilled(result); })
  .catch((error) => { if (!aborted) return onrejected(error); });
};

result = {
  response: null,
  abort: function() {
    aborted = true;
    abortController.abort();
  },
  href: urlString,
  then,
  catch: function(onrejected) {
    return then(undefined, onrejected)
  },
  getHeader(name) {
    return fetchOptions.headers[name];
  }
};

// inside _getFetchOptions
if (!IS_BROWSER) {
  const httpAgent = new http.Agent({ keepAlive: true });
  const httpsAgent = new https.Agent({ keepAlive: true, rejectUnauthorized: this._strictSSL });

  if (referrer) {
    fetchOptions.headers.referer = referrer;
  }
  fetchOptions.agent = (url) => url.protocol == 'http:' ? httpAgent : httpsAgent;
}

it("should use the redirect source URL as the referrer, overriding a provided one", async () => {
  const [requestURL] = await redirectServer("<p>Hello</p>", { "Content-Type": "text/html" });

  const dom = await JSDOM.fromURL(requestURL, { referrer: "http://example.com/" });
  assert.strictEqual(dom.window.document.referrer, requestURL);
});

static fromURL(url, options = {}) {
  return Promise.resolve().then(() => {
    // Remove the hash while sending this through the research loader fetch().
    // It gets added back a few lines down when constructing the JSDOM object.
    const parsedURL = new URL(url);
    const originalHash = parsedURL.hash;
    parsedURL.hash = "";
    url = parsedURL.href;

    options = normalizeFromURLOptions(options);

    const resourceLoader = resourcesToResourceLoader(options.resources);
    const resourceLoaderForInitialRequest = resourceLoader.constructor === NoOpResourceLoader ?
      new ResourceLoader() :
      resourceLoader;

    const req = resourceLoaderForInitialRequest.fetch(url, {
      accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
      cookieJar: options.cookieJar,
      referrer: options.referrer
    });

    return req.then(body => {
      const res = req.response;

      options = Object.assign(options, {
        url: req.href + originalHash,
        contentType: res.headers["content-type"],
        referrer: req.getHeader("referer")
      });

      return new JSDOM(body, options);
    });
  });
}

module.exports = function nodeFetchCookieDecorator (nodeFetch, jar) {
  const fetchCookie = require('./')(nodeFetch, jar)

  return function nodeFetchCookie (url, userOptions = {}) {
    const opts = Object.assign({}, userOptions, { redirect: 'manual' })

    // Forward identical options to wrapped node-fetch but tell to not handle redirection.
    return fetchCookie(url, opts)
      .then(res => {
        const isRedirect = (res.status === 303 || res.status === 301 || res.status === 302 || res.status === 307)

        // Interpret the proprietary "redirect" option in the same way that node-fetch does.
        if (isRedirect && userOptions.redirect !== 'manual' && userOptions.follow !== 0) {
          const statusOpts = {
            // Since the "follow" flag is not relevant for node-fetch in this case,
            // we'll hijack it for our internal bookkeeping.
            follow: userOptions.follow !== undefined ? userOptions.follow - 1 : undefined
          }

          if (res.status !== 307) {
            statusOpts.method = 'GET'
            statusOpts.body = null
          }

          const redirectOpts = Object.assign({}, userOptions, statusOpts)

          return nodeFetchCookie(res.headers.get('location'), redirectOpts)
        } else {
          return res
        }
      })
  }
}

_getFetchOptions({ accept = "*/*" }) {
  /** @type RequestInit */
  const fetchOptions = {};

  fetchOptions.headers = {
    "User-Agent": this._userAgent,
    "Accept-Language": "en",
    "Accept-Encoding": "gzip",
    Accept: accept,
  };

  if (!IS_BROWSER) {
    const httpAgent = new http.Agent({ keepAlive: true });
    const httpsAgent = new https.Agent({ keepAlive: true, rejectUnauthorized: this._strictSSL });
    fetchOptions.agent = (url) => url.protocol == 'http:' ? httpAgent : httpsAgent;
  }

  return fetchOptions;
}

// inside fetch(urlString, options = {})
let lastUrl = options.referrer;
let lastOpts = null;

const myFetch = (url, opts) => {
  if (lastUrl && !IS_BROWSER) {
    opts.headers.referer = lastUrl;
  }
  lastUrl = url;
  lastOpts = opts;
  return fetch(url, opts);
};

const targetFetch = fetchCookie(myFetch, cookieJar);
const fetchOptions = this._getFetchOptions(options);
const abortController = new AbortController();
fetchOptions.signal = abortController.signal;
const fetchPromise = targetFetch(urlString, fetchOptions);

let aborted = false;
let result;
const then = function(onfulfilled, onrejected) {
  return fetchPromise.then((response) => {
    if (!response.ok) {
      throw new Error(`Unexpected status=${response.status} for ${urlString}`);
    }
    result.response = response;
    result.href = lastUrl;
    return response.arrayBuffer();
  })
  .then((arrayBuffer) => typeof Buffer === "undefined" ? arrayBuffer : Buffer.from(arrayBuffer))
  .then((result) => { if (!aborted) return onfulfilled(result); })
  .catch((error) => {
    if (!aborted) {
      if (onrejected) {
        return onrejected(error);
      } else {
        throw error;
      }
    }
  });
};

result = {
  response: null,
  abort: function() {
    aborted = true;
    abortController.abort();
  },
  href: null,
  then,
  catch: function(onrejected) {
    return then(undefined, onrejected)
  },
  getHeader(name) {
    return lastOpts.headers[name];
  }
};

return result;

describe("fromURL", { skipIfBrowser: true }, () => {
  let server;
  let host;
  before(() => {
    return createServer((req, res) => {
      const [, fixture, query] = /^\/([^?]+)(\?.*)?$/.exec(req.url);

      const headers = { "Content-Type": "text/html" };
      if (query === "?charset=csiso88598e") {
        headers["Content-Type"] = "text/html;charset=csiso88598e";
      }

      res.writeHead(200, headers);
      fs.createReadStream(fixturePath(fixture)).pipe(res);
    }).then(s => {
      server = s;
      host = `http://127.0.0.1:${s.address().port}`;
    });
  });

  after(() => server.destroy());

  describe("with no Content-Type header given", () => {
    for (const encodingFixture of Object.keys(encodingFixtures)) {
      const { name, body } = encodingFixtures[encodingFixture];

      it(`should sniff ${encodingFixture} as ${name}`, () => {
        return JSDOM.fromURL(`${host}/${encodingFixture}`).then(dom => {
          assert.strictEqual(dom.window.document.characterSet, name);
          assert.strictEqual(dom.window.document.body.textContent, body);
        });
      });
    }
  });

  describe("with a Content-Type header specifying csiso88598e", () => {
    for (const encodingFixture of Object.keys(encodingFixtures)) {
      const { nameWhenOverridden, bodyWhenOverridden } = encodingFixtures[encodingFixture];

      it(`should sniff ${encodingFixture} as ${nameWhenOverridden}`, () => {
        return JSDOM.fromURL(`${host}/${encodingFixture}?charset=csiso88598e`).then(dom => {
          assert.strictEqual(dom.window.document.characterSet, nameWhenOverridden);
          assert.strictEqual(dom.window.document.contentType, "text/html"); // encoding should be stripped

          if (bodyWhenOverridden) {
            assert.strictEqual(dom.window.document.body.textContent, bodyWhenOverridden);
          }
        });
      });
    }
  });
});

static fromURL(url, options = {}) {
  return Promise.resolve().then(() => {
    return req.then(body => {
      const res = req.response;

      options = Object.assign(options, {
        url: req.href + originalHash,
        contentType: res.headers["content-type"],
        referrer: req.getHeader("referer")
      });

      return new JSDOM(body, options);
    });
  });
}

Headers {
  [Symbol(map)]: [Object: null prototype] {
    'content-type': [ 'text/html;charset=csiso88598e' ],
    date: [ 'Mon, 29 Jun 2020 20:44:07 GMT' ],
    connection: [ 'keep-alive' ],
    'transfer-encoding': [ 'chunked' ]
  }
} undefined

// inside `then`
const { ok, status } = response;
if (!ok) {
  throw new Error(`Unexpected status=${status} for ${urlString}`);
}
const headers = {};
for (const [ key, value ] of response.headers) {
  headers[key] = value;
}

result.response = {
  status,
  headers,
};
result.href = lastUrl;
return response.arrayBuffer();

// inside `fetch(urlString, options = {})`
const cookieJar = options.cookieJar;
if (!cookieJar.__setCookie) {
  cookieJar.__setCookie = cookieJar.setCookie;
  cookieJar.setCookie = (...args) => {
    if (args.length === 3) {
      args.splice(2, 0, {});
    }
    if (args.length === 4) {
      args[2].ignoreError = true;
    }
    return cookieJar.__setCookie(...args);
  }
}

const HttpsProxyAgent = require("https-proxy-agent");

// _getFetchOptions({ accept = "*/*" }) {
if (!IS_BROWSER) {
  const proxyAgent = this._proxy ? new HttpsProxyAgent(this._proxy) : null;
  const httpAgent = new http.Agent({ keepAlive: true });
  const httpsAgent = new https.Agent({ keepAlive: true, rejectUnauthorized: this._strictSSL });
  fetchOptions.agent = (url) => proxyAgent ? proxyAgent : (url.protocol == 'http:' ? httpAgent : httpsAgent);
}

const url = require("url");
const HttpProxyAgent = require("http-proxy-agent");
const HttpsProxyAgent = require("https-proxy-agent");

// _getFetchOptions({ accept = "*/*" }) {
if (!IS_BROWSER) {
  const agentOpts = { keepAlive: true, rejectUnauthorized: this._strictSSL };
  const proxyOpts = { ...agentOpts, ...(this._proxy ? url.parse(this._proxy) : {}) };
  const httpAgent = this._proxy ? new HttpProxyAgent(proxyOpts) : new http.Agent(agentOpts);
  const httpsAgent = this._proxy ? new HttpsProxyAgent(proxyOpts) : new https.Agent(agentOpts);
  fetchOptions.agent = (url) => url.protocol == 'http:' ? httpAgent : httpsAgent;
}

/**
 *
 * @param {string} protocol "http:" or "https:"
 */
_getAgent(protocol) {
  const isHttps = protocol === "https:";
  const agentOpts = { keepAlive: true, rejectUnauthorized: this._strictSSL };
  if (this._proxy) {
    agentOpts.rejectUnauthorized = this._strictSSL;
    const proxyOpts = { ...url.parse(this._proxy), ...agentOpts };
    return isHttps ? new HttpsProxyAgent(proxyOpts) : new HttpProxyAgent(proxyOpts);
  } else {
    return isHttps ? new https.Agent(agentOpts) : new http.Agent(agentOpts);
  }
}

// inside _getFetchOptions({ accept = "*/*" }) {
if (!IS_BROWSER) {
  fetchOptions.agent = (url) => this._getAgent(url.protocol);
}

/**
 *
 * @param {string} protocol "http:" or "https:"
 */
_getAgent(protocol) {
  const isHttps = protocol === "https:";
  const agentOpts = { keepAlive: true, rejectUnauthorized: this._strictSSL };
  if (this._proxy) {
    agentOpts.rejectUnauthorized = this._strictSSL;
    const proxyOpts = { ...url.parse(this._proxy), ...agentOpts };
    return isHttps ? new HttpsProxyAgent(proxyOpts) : new HttpProxyAgent(proxyOpts);
  } else {
    return isHttps ? new https.Agent(agentOpts) : new http.Agent(agentOpts);
  }
}

// inside _getFetchOptions({ accept = "*/*" }) {
case "http":
case "https": {
  const cookieJar = options.cookieJar;
  if (!cookieJar.__setCookie) {
    cookieJar.__setCookie = cookieJar.setCookie;
    cookieJar.setCookie = (...args) => {
      if (args.length === 3) {
        args.splice(2, 0, {});
      }
      if (args.length === 4) {
        args[2].ignoreError = true;
      }
      return cookieJar.__setCookie(...args);
    }
  }

  let lastUrl = options.referrer;
  let lastOpts = null;

  const myFetch = (url, opts) => {
    if (lastUrl && !IS_BROWSER) {
      opts.headers.referer = lastUrl;
    }
    lastUrl = url;
    lastOpts = opts;
    return fetch(url, opts);
  };

  const targetFetch = fetchCookie(myFetch, cookieJar);
  const fetchOptions = this._getFetchOptions(options);
  const abortController = new AbortController();
  fetchOptions.signal = abortController.signal;
  const fetchPromise = targetFetch(urlString, fetchOptions);

  let aborted = false;
  let result;
  const then = function(onfulfilled, onrejected) {
    return fetchPromise.then((response) => {
      const { ok, status } = response;
      if (!ok) {
        throw new Error(`Unexpected status=${status} for ${urlString}`);
      }
      const headers = {};
      for (const [ key, value ] of response.headers) {
        headers[key] = value;
      }

      result.response = {
        status,
        headers,
      };
      result.href = lastUrl;
      return response.arrayBuffer();
    })
    .then((arrayBuffer) => typeof Buffer === "undefined" ? arrayBuffer : Buffer.from(arrayBuffer))
    .then((result) => { if (!aborted) return onfulfilled(result); })
    .catch((error) => {
      if (!aborted) {
        if (onrejected) {
          return onrejected(error);
        } else {
          throw error;
        }
      }
    });
  };

  result = {
    response: null,
    abort: function() {
      aborted = true;
      abortController.abort();
    },
    href: null,
    then,
    catch: function(onrejected) {
      return then(undefined, onrejected)
    },
    getHeader(name) {
      return lastOpts.headers[name];
    }
  };

  return result;
}