yumechi-no-kuni/packages/backend/src/core/HttpRequestService.ts

342 lines
9.7 KiB
TypeScript
Raw Normal View History

2022-09-17 13:27:08 -05:00
import * as http from 'node:http';
import * as https from 'node:https';
import CacheableLookup from 'cacheable-lookup';
import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent';
import { Inject, Injectable } from '@nestjs/common';
import { DI } from '@/di-symbols.js';
2022-09-20 15:33:11 -05:00
import type { Config } from '@/config.js';
2022-09-17 13:27:08 -05:00
import { StatusError } from '@/misc/status-error.js';
import { bindThis } from '@/decorators.js';
import * as undici from 'undici';
import { LookupFunction } from 'node:net';
import { LoggerService } from '@/core/LoggerService.js';
import type Logger from '@/logger.js';
2022-09-17 13:27:08 -05:00
// true to allow, false to deny
export type IpChecker = (ip: string) => boolean;
/*
* Child class to create and save Agent for fetch.
* You should construct this when you want
* to change timeout, size limit, socket connect function, etc.
*/
export class UndiciFetcher {
2022-09-17 13:27:08 -05:00
/**
* Get http non-proxy agent (undici)
2022-09-17 13:27:08 -05:00
*/
public nonProxiedAgent: undici.Agent;
2022-09-17 13:27:08 -05:00
/**
* Get http proxy or non-proxy agent (undici)
2022-09-17 13:27:08 -05:00
*/
public agent: undici.ProxyAgent | undici.Agent;
private proxyBypassHosts: string[];
private userAgent: string | undefined;
private logger: Logger | undefined;
constructor(
args: {
agentOptions: undici.Agent.Options;
proxy?: {
uri: string;
options?: undici.Agent.Options; // Override of agentOptions
},
proxyBypassHosts?: string[];
userAgent?: string;
},
logger?: Logger,
) {
this.logger = logger;
this.logger?.debug('UndiciFetcher constructor', args);
this.proxyBypassHosts = args.proxyBypassHosts ?? [];
this.userAgent = args.userAgent;
this.nonProxiedAgent = new undici.Agent({
...args.agentOptions,
connect: (process.env.NODE_ENV !== 'production' && typeof args.agentOptions.connect !== 'function')
? (options, cb) => {
// Custom connector for debug
undici.buildConnector(args.agentOptions.connect as undici.buildConnector.BuildOptions)(options, (err, socket) => {
this.logger?.debug('Socket connector called', socket);
if (err) {
this.logger?.debug(`Socket error`, err);
cb(new Error(`Error while socket connecting\n${err}`), null);
return;
}
this.logger?.debug(`Socket connected: port ${socket.localPort} => remote ${socket.remoteAddress}`);
cb(null, socket);
});
} : args.agentOptions.connect,
});
this.agent = args.proxy
? new undici.ProxyAgent({
...args.agentOptions,
...args.proxy.options,
uri: args.proxy.uri,
connect: (process.env.NODE_ENV !== 'production' && typeof (args.proxy?.options?.connect ?? args.agentOptions.connect) !== 'function')
? (options, cb) => {
// Custom connector for debug
undici.buildConnector((args.proxy?.options?.connect ?? args.agentOptions.connect) as undici.buildConnector.BuildOptions)(options, (err, socket) => {
this.logger?.debug('Socket connector called (secure)', socket);
if (err) {
this.logger?.debug(`Socket error`, err);
cb(new Error(`Error while socket connecting\n${err}`), null);
return;
}
this.logger?.debug(`Socket connected (secure): port ${socket.localPort} => remote ${socket.remoteAddress}`);
cb(null, socket);
});
} : (args.proxy?.options?.connect ?? args.agentOptions.connect),
})
: this.nonProxiedAgent;
}
2022-09-17 13:27:08 -05:00
/**
* Get agent by URL
* @param url URL
* @param bypassProxy Allways bypass proxy
2022-09-17 13:27:08 -05:00
*/
@bindThis
public getAgentByUrl(url: URL, bypassProxy = false): undici.Agent | undici.ProxyAgent {
if (bypassProxy || this.proxyBypassHosts.includes(url.hostname)) {
return this.nonProxiedAgent;
} else {
return this.agent;
}
}
@bindThis
public async fetch(
url: string | URL,
options: undici.RequestInit = {},
privateOptions: { noOkError?: boolean; bypassProxy?: boolean; } = { noOkError: false, bypassProxy: false }
): Promise<undici.Response> {
const res = await undici.fetch(url, {
dispatcher: this.getAgentByUrl(new URL(url), privateOptions.bypassProxy),
...options,
headers: {
'User-Agent': this.userAgent ?? '',
...(options.headers ?? {}),
},
}).catch((err) => {
this.logger?.error('fetch error', err);
throw new StatusError('Resource Unreachable', 500, 'Resource Unreachable');
});
if (!res.ok && !privateOptions.noOkError) {
throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText);
}
return res;
}
@bindThis
public async getJson<T extends unknown>(url: string, accept = 'application/json, */*', headers?: Record<string, string>): Promise<T> {
const res = await this.fetch(
url,
{
headers: Object.assign({
Accept: accept,
}, headers ?? {}),
}
);
return await res.json() as T;
}
@bindThis
public async getHtml(url: string, accept = 'text/html, */*', headers?: Record<string, string>): Promise<string> {
const res = await this.fetch(
url,
{
headers: Object.assign({
Accept: accept,
}, headers ?? {}),
}
);
return await res.text();
}
}
@Injectable()
export class HttpRequestService {
public defaultFetcher: UndiciFetcher;
public fetch: UndiciFetcher['fetch'];
public getHtml: UndiciFetcher['getHtml'];
public defaultJsonFetcher: UndiciFetcher;
public getJson: UndiciFetcher['getJson'];
//#region for old http/https, only used in S3Service
// http non-proxy agent
private http: http.Agent;
// https non-proxy agent
private https: https.Agent;
// http proxy or non-proxy agent
2022-09-17 13:27:08 -05:00
public httpAgent: http.Agent;
// https proxy or non-proxy agent
2022-09-17 13:27:08 -05:00
public httpsAgent: https.Agent;
//#endregion
public readonly dnsCache: CacheableLookup;
public readonly clientDefaults: undici.Agent.Options;
private maxSockets: number;
private logger: Logger;
2022-09-17 13:27:08 -05:00
constructor(
@Inject(DI.config)
private config: Config,
private loggerService: LoggerService,
2022-09-17 13:27:08 -05:00
) {
this.logger = this.loggerService.getLogger('http-request');
this.dnsCache = new CacheableLookup({
2022-09-17 13:27:08 -05:00
maxTtl: 3600, // 1hours
errorTtl: 30, // 30secs
lookup: false, // nativeのdns.lookupにfallbackしない
});
this.clientDefaults = {
keepAliveTimeout: 30 * 1000,
keepAliveMaxTimeout: 10 * 60 * 1000,
keepAliveTimeoutThreshold: 1 * 1000,
strictContentLength: true,
headersTimeout: 10 * 1000,
bodyTimeout: 10 * 1000,
maxHeaderSize: 16364, // default
maxResponseSize: 10 * 1024 * 1024,
maxRedirections: 3,
connect: {
timeout: 10 * 1000, // コネクションが確立するまでのタイムアウト
maxCachedSessions: 300, // TLSセッションのキャッシュ数 https://github.com/nodejs/undici/blob/v5.14.0/lib/core/connect.js#L80
lookup: this.dnsCache.lookup as LookupFunction, // https://github.com/nodejs/undici/blob/v5.14.0/lib/core/connect.js#L98
},
}
this.maxSockets = Math.max(64, this.config.deliverJobConcurrency ?? 128);
this.defaultFetcher = new UndiciFetcher(this.getStandardUndiciFetcherOption(), this.logger);
this.fetch = this.defaultFetcher.fetch;
this.getHtml = this.defaultFetcher.getHtml;
this.defaultJsonFetcher = new UndiciFetcher(this.getStandardUndiciFetcherOption({
maxResponseSize: 1024 * 256,
}), this.logger);
this.getJson = this.defaultJsonFetcher.getJson;
//#region for old http/https, only used in S3Service
2022-09-18 13:11:50 -05:00
this.http = new http.Agent({
2022-09-17 13:27:08 -05:00
keepAlive: true,
keepAliveMsecs: 30 * 1000,
lookup: this.dnsCache.lookup,
2022-09-17 13:27:08 -05:00
} as http.AgentOptions);
2022-09-18 13:11:50 -05:00
this.https = new https.Agent({
2022-09-17 13:27:08 -05:00
keepAlive: true,
keepAliveMsecs: 30 * 1000,
lookup: this.dnsCache.lookup,
2022-09-17 13:27:08 -05:00
} as https.AgentOptions);
2022-09-17 13:27:08 -05:00
this.httpAgent = config.proxy
? new HttpProxyAgent({
keepAlive: true,
keepAliveMsecs: 30 * 1000,
maxSockets: this.maxSockets,
2022-09-17 13:27:08 -05:00
maxFreeSockets: 256,
scheduling: 'lifo',
proxy: config.proxy,
})
2022-09-18 13:11:50 -05:00
: this.http;
2022-09-17 13:27:08 -05:00
this.httpsAgent = config.proxy
? new HttpsProxyAgent({
keepAlive: true,
keepAliveMsecs: 30 * 1000,
maxSockets: this.maxSockets,
2022-09-17 13:27:08 -05:00
maxFreeSockets: 256,
scheduling: 'lifo',
proxy: config.proxy,
})
2022-09-18 13:11:50 -05:00
: this.https;
//#endregion
}
@bindThis
public getStandardUndiciFetcherOption(opts: undici.Agent.Options = {}, proxyOpts: undici.Agent.Options = {}) {
return {
agentOptions: {
...this.clientDefaults,
...opts,
},
...(this.config.proxy ? {
proxy: {
uri: this.config.proxy,
options: {
connections: this.maxSockets,
...proxyOpts,
}
}
} : {}),
userAgent: this.config.userAgent,
}
2022-09-17 13:27:08 -05:00
}
/**
* Get http agent by URL
2022-09-17 13:27:08 -05:00
* @param url URL
* @param bypassProxy Allways bypass proxy
*/
@bindThis
public getHttpAgentByUrl(url: URL, bypassProxy = false): http.Agent | https.Agent {
2022-09-17 13:27:08 -05:00
if (bypassProxy || (this.config.proxyBypassHosts || []).includes(url.hostname)) {
2022-09-18 13:11:50 -05:00
return url.protocol === 'http:' ? this.http : this.https;
2022-09-17 13:27:08 -05:00
} else {
return url.protocol === 'http:' ? this.httpAgent : this.httpsAgent;
}
}
/**
* check ip
*/
@bindThis
public getConnectorWithIpCheck(connector: undici.buildConnector.connector, checkIp: IpChecker): undici.buildConnector.connectorAsync {
return (options, cb) => {
connector(options, (err, socket) => {
this.logger.debug('Socket connector (with ip checker) called', socket);
if (err) {
this.logger.error(`Socket error`, err)
cb(new Error(`Error while socket connecting\n${err}`), null);
return;
}
2022-09-17 13:27:08 -05:00
if (socket.remoteAddress == undefined) {
this.logger.error(`Socket error: remoteAddress is undefined`);
cb(new Error('remoteAddress is undefined (maybe socket destroyed)'), null);
return;
}
2022-09-17 13:27:08 -05:00
// allow
if (checkIp(socket.remoteAddress)) {
this.logger.debug(`Socket connected (ip ok): ${socket.localPort} => ${socket.remoteAddress}`);
cb(null, socket);
return;
}
2022-09-17 13:27:08 -05:00
this.logger.error('IP is not allowed', socket);
cb(new StatusError('IP is not allowed', 403, 'IP is not allowed'), null);
socket.destroy();
});
};
2022-09-17 13:27:08 -05:00
}
}