diff --git a/.config/example.yml b/.config/example.yml
index 93dea1f254..cabf167fba 100644
--- a/.config/example.yml
+++ b/.config/example.yml
@@ -122,10 +122,12 @@ id: 'aid'
# Proxy for HTTP/HTTPS
#proxy: http://127.0.0.1:3128
-#proxyBypassHosts: [
-# 'example.com',
-# '192.0.2.8'
-#]
+proxyBypassHosts:
+ - api.deepl.com
+ - api-free.deepl.com
+ - www.recaptcha.net
+ - hcaptcha.com
+ - challenges.cloudflare.com
# Proxy for SMTP/SMTPS
#proxySmtp: http://127.0.0.1:3128 # use HTTP/1.1 CONNECT
diff --git a/packages/backend/package.json b/packages/backend/package.json
index 293b630ad6..fbce696a3e 100644
--- a/packages/backend/package.json
+++ b/packages/backend/package.json
@@ -77,7 +77,6 @@
"misskey-js": "0.0.14",
"ms": "3.0.0-canary.1",
"nested-property": "4.0.0",
- "node-fetch": "3.3.0",
"nodemailer": "6.8.0",
"nsfwjs": "2.4.2",
"oauth": "^0.10.0",
@@ -118,6 +117,7 @@
"twemoji-parser": "14.0.0",
"typeorm": "0.3.11",
"ulid": "2.3.0",
+ "undici": "^5.14.0",
"unzipper": "0.10.11",
"uuid": "9.0.0",
"vary": "1.1.2",
@@ -180,6 +180,7 @@
"execa": "6.1.0",
"jest": "29.3.1",
"jest-mock": "^29.3.1",
+ "node-fetch": "3.3.0",
"typescript": "4.9.4"
}
}
diff --git a/packages/backend/src/core/CaptchaService.ts b/packages/backend/src/core/CaptchaService.ts
index 0207cf58a0..1e98914052 100644
--- a/packages/backend/src/core/CaptchaService.ts
+++ b/packages/backend/src/core/CaptchaService.ts
@@ -1,7 +1,4 @@
-import { Inject, Injectable } from '@nestjs/common';
-import { DI } from '@/di-symbols.js';
-import type { UsersRepository } from '@/models/index.js';
-import type { Config } from '@/config.js';
+import { Injectable } from '@nestjs/common';
import { HttpRequestService } from '@/core/HttpRequestService.js';
import { bindThis } from '@/decorators.js';
@@ -13,9 +10,6 @@ type CaptchaResponse = {
@Injectable()
export class CaptchaService {
constructor(
- @Inject(DI.config)
- private config: Config,
-
private httpRequestService: HttpRequestService,
) {
}
@@ -27,16 +21,16 @@ export class CaptchaService {
response,
});
- const res = await fetch(url, {
- method: 'POST',
- body: params,
- headers: {
- 'User-Agent': this.config.userAgent,
+ const res = await this.httpRequestService.fetch(
+ url,
+ {
+ method: 'POST',
+ body: params,
},
- // TODO
- //timeout: 10 * 1000,
- agent: (url, bypassProxy) => this.httpRequestService.getAgentByUrl(url, bypassProxy),
- }).catch(err => {
+ {
+ noOkError: true,
+ }
+ ).catch(err => {
throw `${err.message ?? err}`;
});
diff --git a/packages/backend/src/core/DownloadService.ts b/packages/backend/src/core/DownloadService.ts
index 62123246a7..a3078bff45 100644
--- a/packages/backend/src/core/DownloadService.ts
+++ b/packages/backend/src/core/DownloadService.ts
@@ -8,11 +8,12 @@ import got, * as Got from 'got';
import chalk from 'chalk';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
-import { HttpRequestService } from '@/core/HttpRequestService.js';
+import { HttpRequestService, UndiciFetcher } from '@/core/HttpRequestService.js';
import { createTemp } from '@/misc/create-temp.js';
import { StatusError } from '@/misc/status-error.js';
import { LoggerService } from '@/core/LoggerService.js';
import type Logger from '@/logger.js';
+import { buildConnector } from 'undici';
const pipeline = util.promisify(stream.pipeline);
import { bindThis } from '@/decorators.js';
@@ -20,6 +21,7 @@ import { bindThis } from '@/decorators.js';
@Injectable()
export class DownloadService {
private logger: Logger;
+ private undiciFetcher: UndiciFetcher;
constructor(
@Inject(DI.config)
@@ -29,70 +31,42 @@ export class DownloadService {
private loggerService: LoggerService,
) {
this.logger = this.loggerService.getLogger('download');
+
+ this.undiciFetcher = new UndiciFetcher(this.httpRequestService.getStandardUndiciFetcherOption(
+ {
+ connect: process.env.NODE_ENV === 'development' ?
+ this.httpRequestService.clientDefaults.connect
+ :
+ this.httpRequestService.getConnectorWithIpCheck(
+ buildConnector({
+ ...this.httpRequestService.clientDefaults.connect,
+ }),
+ (ip) => !this.isPrivateIp(ip)
+ ),
+ bodyTimeout: 30 * 1000,
+ },
+ {
+ connect: this.httpRequestService.clientDefaults.connect,
+ }
+ ), this.logger);
}
@bindThis
public async downloadUrl(url: string, path: string): Promise<void> {
this.logger.info(`Downloading ${chalk.cyan(url)} to ${chalk.cyanBright(path)} ...`);
-
+
const timeout = 30 * 1000;
const operationTimeout = 60 * 1000;
const maxSize = this.config.maxFileSize ?? 262144000;
-
- const req = got.stream(url, {
- headers: {
- 'User-Agent': this.config.userAgent,
- },
- timeout: {
- lookup: timeout,
- connect: timeout,
- secureConnect: timeout,
- socket: timeout, // read timeout
- response: timeout,
- send: timeout,
- request: operationTimeout, // whole operation timeout
- },
- agent: {
- http: this.httpRequestService.httpAgent,
- https: this.httpRequestService.httpsAgent,
- },
- http2: false, // default
- retry: {
- limit: 0,
- },
- }).on('response', (res: Got.Response) => {
- if ((process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'test') && !this.config.proxy && res.ip) {
- if (this.isPrivateIp(res.ip)) {
- this.logger.warn(`Blocked address: ${res.ip}`);
- req.destroy();
- }
- }
-
- const contentLength = res.headers['content-length'];
- if (contentLength != null) {
- const size = Number(contentLength);
- if (size > maxSize) {
- this.logger.warn(`maxSize exceeded (${size} > ${maxSize}) on response`);
- req.destroy();
- }
- }
- }).on('downloadProgress', (progress: Got.Progress) => {
- if (progress.transferred > maxSize) {
- this.logger.warn(`maxSize exceeded (${progress.transferred} > ${maxSize}) on downloadProgress`);
- req.destroy();
- }
- });
-
- try {
- await pipeline(req, fs.createWriteStream(path));
- } catch (e) {
- if (e instanceof Got.HTTPError) {
- throw new StatusError(`${e.response.statusCode} ${e.response.statusMessage}`, e.response.statusCode, e.response.statusMessage);
- } else {
- throw e;
- }
+
+ const response = await this.undiciFetcher.fetch(url);
+
+ if (response.body === null) {
+ throw new StatusError('No body', 400, 'No body');
}
-
+
+ await pipeline(stream.Readable.fromWeb(response.body), fs.createWriteStream(path));
+
this.logger.succ(`Download finished: ${chalk.cyan(url)}`);
}
@@ -114,7 +88,7 @@ export class DownloadService {
cleanup();
}
}
-
+
@bindThis
private isPrivateIp(ip: string): boolean {
for (const net of this.config.allowedPrivateNetworks ?? []) {
@@ -124,6 +98,6 @@ export class DownloadService {
}
}
- return PrivateIp(ip);
+ return PrivateIp(ip) ?? false;
}
}
diff --git a/packages/backend/src/core/DriveService.ts b/packages/backend/src/core/DriveService.ts
index 5954abba91..747dfd6130 100644
--- a/packages/backend/src/core/DriveService.ts
+++ b/packages/backend/src/core/DriveService.ts
@@ -375,8 +375,19 @@ export class DriveService {
partSize: s3.endpoint.hostname === 'storage.googleapis.com' ? 500 * 1024 * 1024 : 8 * 1024 * 1024,
});
- const result = await upload.promise();
- if (result) this.registerLogger.debug(`Uploaded: ${result.Bucket}/${result.Key} => ${result.Location}`);
+ await upload.promise()
+ .then(
+ result => {
+ if (result) {
+ this.registerLogger.debug(`Uploaded: ${result.Bucket}/${result.Key} => ${result.Location}`);
+ } else {
+ this.registerLogger.error(`Upload Result Empty: key = ${key}, filename = ${filename}`);
+ }
+ },
+ err => {
+ this.registerLogger.error(`Upload Failed: key = ${key}, filename = ${filename}`, err);
+ }
+ );
}
@bindThis
@@ -462,6 +473,8 @@ export class DriveService {
}
}
+ this.registerLogger.debug(`ADD DRIVE FILE: user ${user?.id ?? 'not set'}, name ${detectedName}, tmp ${path}`);
+
//#region Check drive usage
if (user && !isLink) {
const usage = await this.driveFileEntityService.calcDriveUsageOf(user);
diff --git a/packages/backend/src/core/FetchInstanceMetadataService.ts b/packages/backend/src/core/FetchInstanceMetadataService.ts
index 7eea45200e..cb9d099a22 100644
--- a/packages/backend/src/core/FetchInstanceMetadataService.ts
+++ b/packages/backend/src/core/FetchInstanceMetadataService.ts
@@ -1,7 +1,6 @@
import { URL } from 'node:url';
import { Inject, Injectable } from '@nestjs/common';
import { JSDOM } from 'jsdom';
-import fetch from 'node-fetch';
import tinycolor from 'tinycolor2';
import type { Instance } from '@/models/entities/Instance.js';
import type { InstancesRepository } from '@/models/index.js';
@@ -191,11 +190,7 @@ export class FetchInstanceMetadataService {
const faviconUrl = url + '/favicon.ico';
- const favicon = await fetch(faviconUrl, {
- // TODO
- //timeout: 10000,
- agent: url => this.httpRequestService.getAgentByUrl(url),
- });
+ const favicon = await this.httpRequestService.fetch(faviconUrl, {}, { noOkError: true });
if (favicon.ok) {
return faviconUrl;
diff --git a/packages/backend/src/core/HttpRequestService.ts b/packages/backend/src/core/HttpRequestService.ts
index 49b28ae523..8639b5713d 100644
--- a/packages/backend/src/core/HttpRequestService.ts
+++ b/packages/backend/src/core/HttpRequestService.ts
@@ -1,67 +1,257 @@
import * as http from 'node:http';
import * as https from 'node:https';
import CacheableLookup from 'cacheable-lookup';
-import fetch from 'node-fetch';
import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent';
import { Inject, Injectable } from '@nestjs/common';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import { StatusError } from '@/misc/status-error.js';
import { bindThis } from '@/decorators.js';
-import type { Response } from 'node-fetch';
-import type { URL } from 'node:url';
+import * as undici from 'undici';
+import { LookupFunction } from 'node:net';
+import { LoggerService } from '@/core/LoggerService.js';
+import type Logger from '@/logger.js';
+
+// true to allow, false to deny
+export type IpChecker = (ip: string) => boolean;
+
+/*
+ * Child class to create and save Agent for fetch.
+ * You should construct this when you want
+ * to change timeout, size limit, socket connect function, etc.
+ */
+export class UndiciFetcher {
+ /**
+ * Get http non-proxy agent (undici)
+ */
+ public nonProxiedAgent: undici.Agent;
+
+ /**
+ * Get http proxy or non-proxy agent (undici)
+ */
+ public agent: undici.ProxyAgent | undici.Agent;
+
+ private proxyBypassHosts: string[];
+ private userAgent: string | undefined;
+
+ private logger: Logger | undefined;
+
+ constructor(
+ args: {
+ agentOptions: undici.Agent.Options;
+ proxy?: {
+ uri: string;
+ options?: undici.Agent.Options; // Override of agentOptions
+ },
+ proxyBypassHosts?: string[];
+ userAgent?: string;
+ },
+ logger?: Logger,
+ ) {
+ this.logger = logger;
+ this.logger?.debug('UndiciFetcher constructor', args);
+
+ this.proxyBypassHosts = args.proxyBypassHosts ?? [];
+ this.userAgent = args.userAgent;
+
+ this.nonProxiedAgent = new undici.Agent({
+ ...args.agentOptions,
+ connect: (process.env.NODE_ENV !== 'production' && typeof args.agentOptions.connect !== 'function')
+ ? (options, cb) => {
+ // Custom connector for debug
+ undici.buildConnector(args.agentOptions.connect as undici.buildConnector.BuildOptions)(options, (err, socket) => {
+ this.logger?.debug('Socket connector called', socket);
+ if (err) {
+ this.logger?.debug(`Socket error`, err);
+ cb(new Error(`Error while socket connecting\n${err}`), null);
+ return;
+ }
+ this.logger?.debug(`Socket connected: port ${socket.localPort} => remote ${socket.remoteAddress}`);
+ cb(null, socket);
+ });
+ } : args.agentOptions.connect,
+ });
+
+ this.agent = args.proxy
+ ? new undici.ProxyAgent({
+ ...args.agentOptions,
+ ...args.proxy.options,
+
+ uri: args.proxy.uri,
+
+ connect: (process.env.NODE_ENV !== 'production' && typeof (args.proxy?.options?.connect ?? args.agentOptions.connect) !== 'function')
+ ? (options, cb) => {
+ // Custom connector for debug
+ undici.buildConnector((args.proxy?.options?.connect ?? args.agentOptions.connect) as undici.buildConnector.BuildOptions)(options, (err, socket) => {
+ this.logger?.debug('Socket connector called (secure)', socket);
+ if (err) {
+ this.logger?.debug(`Socket error`, err);
+ cb(new Error(`Error while socket connecting\n${err}`), null);
+ return;
+ }
+ this.logger?.debug(`Socket connected (secure): port ${socket.localPort} => remote ${socket.remoteAddress}`);
+ cb(null, socket);
+ });
+ } : (args.proxy?.options?.connect ?? args.agentOptions.connect),
+ })
+ : this.nonProxiedAgent;
+ }
+
+ /**
+ * Get agent by URL
+ * @param url URL
+ * @param bypassProxy Allways bypass proxy
+ */
+ @bindThis
+ public getAgentByUrl(url: URL, bypassProxy = false): undici.Agent | undici.ProxyAgent {
+ if (bypassProxy || this.proxyBypassHosts.includes(url.hostname)) {
+ return this.nonProxiedAgent;
+ } else {
+ return this.agent;
+ }
+ }
+
+ @bindThis
+ public async fetch(
+ url: string | URL,
+ options: undici.RequestInit = {},
+ privateOptions: { noOkError?: boolean; bypassProxy?: boolean; } = { noOkError: false, bypassProxy: false }
+ ): Promise<undici.Response> {
+ const res = await undici.fetch(url, {
+ dispatcher: this.getAgentByUrl(new URL(url), privateOptions.bypassProxy),
+ ...options,
+ headers: {
+ 'User-Agent': this.userAgent ?? '',
+ ...(options.headers ?? {}),
+ },
+ }).catch((err) => {
+ this.logger?.error('fetch error', err);
+ throw new StatusError('Resource Unreachable', 500, 'Resource Unreachable');
+ });
+ if (!res.ok && !privateOptions.noOkError) {
+ throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText);
+ }
+ return res;
+ }
+
+ @bindThis
+ public async getJson<T extends unknown>(url: string, accept = 'application/json, */*', headers?: Record<string, string>): Promise<T> {
+ const res = await this.fetch(
+ url,
+ {
+ headers: Object.assign({
+ Accept: accept,
+ }, headers ?? {}),
+ }
+ );
+
+ return await res.json() as T;
+ }
+
+ @bindThis
+ public async getHtml(url: string, accept = 'text/html, */*', headers?: Record<string, string>): Promise<string> {
+ const res = await this.fetch(
+ url,
+ {
+ headers: Object.assign({
+ Accept: accept,
+ }, headers ?? {}),
+ }
+ );
+
+ return await res.text();
+ }
+}
@Injectable()
export class HttpRequestService {
- /**
- * Get http non-proxy agent
- */
+ public defaultFetcher: UndiciFetcher;
+ public fetch: UndiciFetcher['fetch'];
+ public getHtml: UndiciFetcher['getHtml'];
+ public defaultJsonFetcher: UndiciFetcher;
+ public getJson: UndiciFetcher['getJson'];
+
+ //#region for old http/https, only used in S3Service
+ // http non-proxy agent
private http: http.Agent;
- /**
- * Get https non-proxy agent
- */
+ // https non-proxy agent
private https: https.Agent;
- /**
- * Get http proxy or non-proxy agent
- */
+ // http proxy or non-proxy agent
public httpAgent: http.Agent;
- /**
- * Get https proxy or non-proxy agent
- */
+ // https proxy or non-proxy agent
public httpsAgent: https.Agent;
+ //#endregion
+
+ public readonly dnsCache: CacheableLookup;
+ public readonly clientDefaults: undici.Agent.Options;
+ private maxSockets: number;
+
+ private logger: Logger;
constructor(
@Inject(DI.config)
private config: Config,
+ private loggerService: LoggerService,
) {
- const cache = new CacheableLookup({
+ this.logger = this.loggerService.getLogger('http-request');
+
+ this.dnsCache = new CacheableLookup({
maxTtl: 3600, // 1hours
errorTtl: 30, // 30secs
lookup: false, // nativeのdns.lookupにfallbackしない
});
-
+
+ this.clientDefaults = {
+ keepAliveTimeout: 30 * 1000,
+ keepAliveMaxTimeout: 10 * 60 * 1000,
+ keepAliveTimeoutThreshold: 1 * 1000,
+ strictContentLength: true,
+ headersTimeout: 10 * 1000,
+ bodyTimeout: 10 * 1000,
+ maxHeaderSize: 16364, // default
+ maxResponseSize: 10 * 1024 * 1024,
+ maxRedirections: 3,
+ connect: {
+ timeout: 10 * 1000, // コネクションが確立するまでのタイムアウト
+ maxCachedSessions: 300, // TLSセッションのキャッシュ数 https://github.com/nodejs/undici/blob/v5.14.0/lib/core/connect.js#L80
+ lookup: this.dnsCache.lookup as LookupFunction, // https://github.com/nodejs/undici/blob/v5.14.0/lib/core/connect.js#L98
+ },
+ }
+
+ this.maxSockets = Math.max(64, this.config.deliverJobConcurrency ?? 128);
+
+ this.defaultFetcher = new UndiciFetcher(this.getStandardUndiciFetcherOption(), this.logger);
+
+ this.fetch = this.defaultFetcher.fetch;
+ this.getHtml = this.defaultFetcher.getHtml;
+
+ this.defaultJsonFetcher = new UndiciFetcher(this.getStandardUndiciFetcherOption({
+ maxResponseSize: 1024 * 256,
+ }), this.logger);
+
+ this.getJson = this.defaultJsonFetcher.getJson;
+
+ //#region for old http/https, only used in S3Service
this.http = new http.Agent({
keepAlive: true,
keepAliveMsecs: 30 * 1000,
- lookup: cache.lookup,
+ lookup: this.dnsCache.lookup,
} as http.AgentOptions);
this.https = new https.Agent({
keepAlive: true,
keepAliveMsecs: 30 * 1000,
- lookup: cache.lookup,
+ lookup: this.dnsCache.lookup,
} as https.AgentOptions);
-
- const maxSockets = Math.max(256, config.deliverJobConcurrency ?? 128);
-
+
this.httpAgent = config.proxy
? new HttpProxyAgent({
keepAlive: true,
keepAliveMsecs: 30 * 1000,
- maxSockets,
+ maxSockets: this.maxSockets,
maxFreeSockets: 256,
scheduling: 'lifo',
proxy: config.proxy,
@@ -72,21 +262,42 @@ export class HttpRequestService {
? new HttpsProxyAgent({
keepAlive: true,
keepAliveMsecs: 30 * 1000,
- maxSockets,
+ maxSockets: this.maxSockets,
maxFreeSockets: 256,
scheduling: 'lifo',
proxy: config.proxy,
})
: this.https;
+ //#endregion
+ }
+
+ @bindThis
+ public getStandardUndiciFetcherOption(opts: undici.Agent.Options = {}, proxyOpts: undici.Agent.Options = {}) {
+ return {
+ agentOptions: {
+ ...this.clientDefaults,
+ ...opts,
+ },
+ ...(this.config.proxy ? {
+ proxy: {
+ uri: this.config.proxy,
+ options: {
+ connections: this.maxSockets,
+ ...proxyOpts,
+ }
+ }
+ } : {}),
+ userAgent: this.config.userAgent,
+ }
}
/**
- * Get agent by URL
+ * Get http agent by URL
* @param url URL
* @param bypassProxy Allways bypass proxy
*/
@bindThis
- public getAgentByUrl(url: URL, bypassProxy = false): http.Agent | https.Agent {
+ public getHttpAgentByUrl(url: URL, bypassProxy = false): http.Agent | https.Agent {
if (bypassProxy || (this.config.proxyBypassHosts || []).includes(url.hostname)) {
return url.protocol === 'http:' ? this.http : this.https;
} else {
@@ -94,67 +305,37 @@ export class HttpRequestService {
}
}
+ /**
+ * check ip
+ */
@bindThis
- public async getJson(url: string, accept = 'application/json, */*', timeout = 10000, headers?: Record<string, string>): Promise<unknown> {
- const res = await this.getResponse({
- url,
- method: 'GET',
- headers: Object.assign({
- 'User-Agent': this.config.userAgent,
- Accept: accept,
- }, headers ?? {}),
- timeout,
- size: 1024 * 256,
- });
+ public getConnectorWithIpCheck(connector: undici.buildConnector.connector, checkIp: IpChecker): undici.buildConnector.connectorAsync {
+ return (options, cb) => {
+ connector(options, (err, socket) => {
+ this.logger.debug('Socket connector (with ip checker) called', socket);
+ if (err) {
+ this.logger.error(`Socket error`, err)
+ cb(new Error(`Error while socket connecting\n${err}`), null);
+ return;
+ }
- return await res.json();
- }
+ if (socket.remoteAddress == undefined) {
+ this.logger.error(`Socket error: remoteAddress is undefined`);
+ cb(new Error('remoteAddress is undefined (maybe socket destroyed)'), null);
+ return;
+ }
- @bindThis
- public async getHtml(url: string, accept = 'text/html, */*', timeout = 10000, headers?: Record<string, string>): Promise<string> {
- const res = await this.getResponse({
- url,
- method: 'GET',
- headers: Object.assign({
- 'User-Agent': this.config.userAgent,
- Accept: accept,
- }, headers ?? {}),
- timeout,
- });
+ // allow
+ if (checkIp(socket.remoteAddress)) {
+ this.logger.debug(`Socket connected (ip ok): ${socket.localPort} => ${socket.remoteAddress}`);
+ cb(null, socket);
+ return;
+ }
- return await res.text();
- }
-
- @bindThis
- public async getResponse(args: {
- url: string,
- method: string,
- body?: string,
- headers: Record<string, string>,
- timeout?: number,
- size?: number,
- }): Promise<Response> {
- const timeout = args.timeout ?? 10 * 1000;
-
- const controller = new AbortController();
- setTimeout(() => {
- controller.abort();
- }, timeout * 6);
-
- const res = await fetch(args.url, {
- method: args.method,
- headers: args.headers,
- body: args.body,
- timeout,
- size: args.size ?? 10 * 1024 * 1024,
- agent: (url) => this.getAgentByUrl(url),
- signal: controller.signal,
- });
-
- if (!res.ok) {
- throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText);
- }
-
- return res;
+ this.logger.error('IP is not allowed', socket);
+ cb(new StatusError('IP is not allowed', 403, 'IP is not allowed'), null);
+ socket.destroy();
+ });
+ };
}
}
diff --git a/packages/backend/src/core/S3Service.ts b/packages/backend/src/core/S3Service.ts
index 0ce69aaa74..930188ce6e 100644
--- a/packages/backend/src/core/S3Service.ts
+++ b/packages/backend/src/core/S3Service.ts
@@ -33,7 +33,7 @@ export class S3Service {
? false
: meta.objectStorageS3ForcePathStyle,
httpOptions: {
- agent: this.httpRequestService.getAgentByUrl(new URL(u), !meta.objectStorageUseProxy),
+ agent: this.httpRequestService.getHttpAgentByUrl(new URL(u), !meta.objectStorageUseProxy),
},
});
}
diff --git a/packages/backend/src/core/WebfingerService.ts b/packages/backend/src/core/WebfingerService.ts
index 4c91ab8438..69df2d0c1b 100644
--- a/packages/backend/src/core/WebfingerService.ts
+++ b/packages/backend/src/core/WebfingerService.ts
@@ -30,7 +30,7 @@ export class WebfingerService {
public async webfinger(query: string): Promise<IWebFinger> {
const url = this.genUrl(query);
- return await this.httpRequestService.getJson(url, 'application/jrd+json, application/json') as IWebFinger;
+ return await this.httpRequestService.getJson<IWebFinger>(url, 'application/jrd+json, application/json');
}
@bindThis
diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts
index d1edd579fa..d44d06a442 100644
--- a/packages/backend/src/core/activitypub/ApRequestService.ts
+++ b/packages/backend/src/core/activitypub/ApRequestService.ts
@@ -5,8 +5,10 @@ import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import type { User } from '@/models/entities/User.js';
import { UserKeypairStoreService } from '@/core/UserKeypairStoreService.js';
-import { HttpRequestService } from '@/core/HttpRequestService.js';
+import { HttpRequestService, UndiciFetcher } from '@/core/HttpRequestService.js';
+import { LoggerService } from '@/core/LoggerService.js';
import { bindThis } from '@/decorators.js';
+import type Logger from '@/logger.js';
type Request = {
url: string;
@@ -28,13 +30,21 @@ type PrivateKey = {
@Injectable()
export class ApRequestService {
+ private undiciFetcher: UndiciFetcher;
+ private logger: Logger;
+
constructor(
@Inject(DI.config)
private config: Config,
private userKeypairStoreService: UserKeypairStoreService,
private httpRequestService: HttpRequestService,
+ private loggerService: LoggerService,
) {
+ this.logger = this.loggerService?.getLogger('ap-request'); // なぜか TypeError: Cannot read properties of undefined (reading 'getLogger') と言われる
+ this.undiciFetcher = new UndiciFetcher(this.httpRequestService.getStandardUndiciFetcherOption({
+ maxRedirections: 0,
+ }), this.logger );
}
@bindThis
@@ -148,16 +158,17 @@ export class ApRequestService {
url,
body,
additionalHeaders: {
- 'User-Agent': this.config.userAgent,
},
});
- await this.httpRequestService.getResponse({
+ await this.undiciFetcher.fetch(
url,
- method: req.request.method,
- headers: req.request.headers,
- body,
- });
+ {
+ method: req.request.method,
+ headers: req.request.headers,
+ body,
+ }
+ );
}
/**
@@ -176,15 +187,16 @@ export class ApRequestService {
},
url,
additionalHeaders: {
- 'User-Agent': this.config.userAgent,
},
});
- const res = await this.httpRequestService.getResponse({
+ const res = await this.httpRequestService.fetch(
url,
- method: req.request.method,
- headers: req.request.headers,
- });
+ {
+ method: req.request.method,
+ headers: req.request.headers,
+ }
+ );
return await res.json();
}
diff --git a/packages/backend/src/core/activitypub/ApResolverService.ts b/packages/backend/src/core/activitypub/ApResolverService.ts
index e96c84f148..dc84f06a6a 100644
--- a/packages/backend/src/core/activitypub/ApResolverService.ts
+++ b/packages/backend/src/core/activitypub/ApResolverService.ts
@@ -4,7 +4,7 @@ import { InstanceActorService } from '@/core/InstanceActorService.js';
import type { NotesRepository, PollsRepository, NoteReactionsRepository, UsersRepository } from '@/models/index.js';
import type { Config } from '@/config.js';
import { MetaService } from '@/core/MetaService.js';
-import { HttpRequestService } from '@/core/HttpRequestService.js';
+import { HttpRequestService, UndiciFetcher } from '@/core/HttpRequestService.js';
import { DI } from '@/di-symbols.js';
import { UtilityService } from '@/core/UtilityService.js';
import { bindThis } from '@/decorators.js';
@@ -12,11 +12,15 @@ import { isCollectionOrOrderedCollection } from './type.js';
import { ApDbResolverService } from './ApDbResolverService.js';
import { ApRendererService } from './ApRendererService.js';
import { ApRequestService } from './ApRequestService.js';
+import { LoggerService } from '@/core/LoggerService.js';
import type { IObject, ICollection, IOrderedCollection } from './type.js';
+import type Logger from '@/logger.js';
export class Resolver {
private history: Set<string>;
private user?: ILocalUser;
+ private undiciFetcher: UndiciFetcher;
+ private logger: Logger;
constructor(
private config: Config,
@@ -31,9 +35,14 @@ export class Resolver {
private httpRequestService: HttpRequestService,
private apRendererService: ApRendererService,
private apDbResolverService: ApDbResolverService,
+ private loggerService: LoggerService,
private recursionLimit = 100,
) {
this.history = new Set();
+ this.logger = this.loggerService?.getLogger('ap-resolve'); // なぜか TypeError: Cannot read properties of undefined (reading 'getLogger') と言われる
+ this.undiciFetcher = new UndiciFetcher(this.httpRequestService.getStandardUndiciFetcherOption({
+ maxRedirections: 0,
+ }), this.logger);
}
@bindThis
@@ -96,8 +105,8 @@ export class Resolver {
}
const object = (this.user
- ? await this.apRequestService.signedGet(value, this.user)
- : await this.httpRequestService.getJson(value, 'application/activity+json, application/ld+json')) as IObject;
+ ? await this.apRequestService.signedGet(value, this.user) as IObject
+ : await this.undiciFetcher.getJson<IObject>(value, 'application/activity+json, application/ld+json'));
if (object == null || (
Array.isArray(object['@context']) ?
diff --git a/packages/backend/src/core/activitypub/LdSignatureService.ts b/packages/backend/src/core/activitypub/LdSignatureService.ts
index b71320ed0b..4e4b7dce2d 100644
--- a/packages/backend/src/core/activitypub/LdSignatureService.ts
+++ b/packages/backend/src/core/activitypub/LdSignatureService.ts
@@ -1,6 +1,5 @@
import * as crypto from 'node:crypto';
import { Inject, Injectable } from '@nestjs/common';
-import fetch from 'node-fetch';
import { HttpRequestService } from '@/core/HttpRequestService.js';
import { bindThis } from '@/decorators.js';
import { CONTEXTS } from './misc/contexts.js';
@@ -116,14 +115,19 @@ class LdSignature {
@bindThis
private async fetchDocument(url: string) {
- const json = await fetch(url, {
- headers: {
- Accept: 'application/ld+json, application/json',
+ const json = await this.httpRequestService.fetch(
+ url,
+ {
+ headers: {
+ Accept: 'application/ld+json, application/json',
+ },
+ // TODO
+ //timeout: this.loderTimeout,
},
- // TODO
- //timeout: this.loderTimeout,
- agent: u => u.protocol === 'http:' ? this.httpRequestService.httpAgent : this.httpRequestService.httpsAgent,
- }).then(res => {
+ {
+ noOkError: true,
+ }
+ ).then(res => {
if (!res.ok) {
throw `${res.status} ${res.statusText}`;
} else {
diff --git a/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts b/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts
index 183ef07477..f0543a5ed1 100644
--- a/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts
+++ b/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts
@@ -33,24 +33,26 @@ export class WebhookDeliverProcessorService {
try {
this.logger.debug(`delivering ${job.data.webhookId}`);
- const res = await this.httpRequestService.getResponse({
- url: job.data.to,
- method: 'POST',
- headers: {
- 'User-Agent': 'Misskey-Hooks',
- 'X-Misskey-Host': this.config.host,
- 'X-Misskey-Hook-Id': job.data.webhookId,
- 'X-Misskey-Hook-Secret': job.data.secret,
- },
- body: JSON.stringify({
- hookId: job.data.webhookId,
- userId: job.data.userId,
- eventId: job.data.eventId,
- createdAt: job.data.createdAt,
- type: job.data.type,
- body: job.data.content,
- }),
- });
+ const res = await this.httpRequestService.fetch(
+ job.data.to,
+ {
+ method: 'POST',
+ headers: {
+ 'User-Agent': 'Misskey-Hooks',
+ 'X-Misskey-Host': this.config.host,
+ 'X-Misskey-Hook-Id': job.data.webhookId,
+ 'X-Misskey-Hook-Secret': job.data.secret,
+ },
+ body: JSON.stringify({
+ hookId: job.data.webhookId,
+ userId: job.data.userId,
+ eventId: job.data.eventId,
+ createdAt: job.data.createdAt,
+ type: job.data.type,
+ body: job.data.content,
+ }),
+ }
+ );
this.webhooksRepository.update({ id: job.data.webhookId }, {
latestSentAt: new Date(),
diff --git a/packages/backend/src/server/api/endpoints/fetch-rss.ts b/packages/backend/src/server/api/endpoints/fetch-rss.ts
index 58fa01ac48..ae6a87513d 100644
--- a/packages/backend/src/server/api/endpoints/fetch-rss.ts
+++ b/packages/backend/src/server/api/endpoints/fetch-rss.ts
@@ -33,15 +33,16 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
private httpRequestService: HttpRequestService,
) {
super(meta, paramDef, async (ps, me) => {
- const res = await this.httpRequestService.getResponse({
- url: ps.url,
- method: 'GET',
- headers: Object.assign({
- 'User-Agent': config.userAgent,
- Accept: 'application/rss+xml, */*',
- }),
- timeout: 5000,
- });
+ const res = await this.httpRequestService.fetch(
+ ps.url,
+ {
+ method: 'GET',
+ headers: {
+ Accept: 'application/rss+xml, */*',
+ },
+ // timeout: 5000,
+ }
+ );
const text = await res.text();
diff --git a/packages/backend/src/server/api/endpoints/notes/translate.ts b/packages/backend/src/server/api/endpoints/notes/translate.ts
index ec16965998..ab19771676 100644
--- a/packages/backend/src/server/api/endpoints/notes/translate.ts
+++ b/packages/backend/src/server/api/endpoints/notes/translate.ts
@@ -1,5 +1,4 @@
import { URLSearchParams } from 'node:url';
-import fetch from 'node-fetch';
import { Inject, Injectable } from '@nestjs/common';
import type { NotesRepository } from '@/models/index.js';
import { Endpoint } from '@/server/api/endpoint-base.js';
@@ -84,25 +83,27 @@ export default class extends Endpoint<typeof meta, typeof paramDef> {
const endpoint = instance.deeplIsPro ? 'https://api.deepl.com/v2/translate' : 'https://api-free.deepl.com/v2/translate';
- const res = await fetch(endpoint, {
- method: 'POST',
- headers: {
- 'Content-Type': 'application/x-www-form-urlencoded',
- 'User-Agent': config.userAgent,
- Accept: 'application/json, */*',
+ const res = await this.httpRequestService.fetch(
+ endpoint,
+ {
+ method: 'POST',
+ headers: {
+ 'Content-Type': 'application/x-www-form-urlencoded',
+ Accept: 'application/json, */*',
+ },
+ body: params.toString(),
},
- body: params,
- // TODO
- //timeout: 10000,
- agent: (url) => this.httpRequestService.getAgentByUrl(url),
- });
+ {
+ noOkError: false,
+ }
+ );
const json = (await res.json()) as {
- translations: {
- detected_source_language: string;
- text: string;
- }[];
- };
+ translations: {
+ detected_source_language: string;
+ text: string;
+ }[];
+ };
return {
sourceLang: json.translations[0].detected_source_language,
diff --git a/packages/backend/src/server/api/integration/DiscordServerService.ts b/packages/backend/src/server/api/integration/DiscordServerService.ts
index 805056da8b..0ac2733817 100644
--- a/packages/backend/src/server/api/integration/DiscordServerService.ts
+++ b/packages/backend/src/server/api/integration/DiscordServerService.ts
@@ -181,7 +181,7 @@ export class DiscordServerService {
}
}));
- const { id, username, discriminator } = (await this.httpRequestService.getJson('https://discord.com/api/users/@me', '*/*', 10 * 1000, {
+ const { id, username, discriminator } = (await this.httpRequestService.getJson('https://discord.com/api/users/@me', '*/*', {
'Authorization': `Bearer ${accessToken}`,
})) as Record<string, unknown>;
@@ -249,7 +249,7 @@ export class DiscordServerService {
}
}));
- const { id, username, discriminator } = (await this.httpRequestService.getJson('https://discord.com/api/users/@me', '*/*', 10 * 1000, {
+ const { id, username, discriminator } = (await this.httpRequestService.getJson('https://discord.com/api/users/@me', '*/*', {
'Authorization': `Bearer ${accessToken}`,
})) as Record<string, unknown>;
if (typeof id !== 'string' || typeof username !== 'string' || typeof discriminator !== 'string') {
diff --git a/packages/backend/src/server/api/integration/GithubServerService.ts b/packages/backend/src/server/api/integration/GithubServerService.ts
index 6f38c262a1..a8c745d2dc 100644
--- a/packages/backend/src/server/api/integration/GithubServerService.ts
+++ b/packages/backend/src/server/api/integration/GithubServerService.ts
@@ -174,7 +174,7 @@ export class GithubServerService {
}
}));
- const { login, id } = (await this.httpRequestService.getJson('https://api.github.com/user', 'application/vnd.github.v3+json', 10 * 1000, {
+ const { login, id } = (await this.httpRequestService.getJson('https://api.github.com/user', 'application/vnd.github.v3+json', {
'Authorization': `bearer ${accessToken}`,
})) as Record<string, unknown>;
if (typeof login !== 'string' || typeof id !== 'string') {
@@ -223,7 +223,7 @@ export class GithubServerService {
}
}));
- const { login, id } = (await this.httpRequestService.getJson('https://api.github.com/user', 'application/vnd.github.v3+json', 10 * 1000, {
+ const { login, id } = (await this.httpRequestService.getJson('https://api.github.com/user', 'application/vnd.github.v3+json', {
'Authorization': `bearer ${accessToken}`,
})) as Record<string, unknown>;
diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts
index baef8fa993..802b404ce6 100644
--- a/packages/backend/src/server/web/UrlPreviewService.ts
+++ b/packages/backend/src/server/web/UrlPreviewService.ts
@@ -63,9 +63,8 @@ export class UrlPreviewService {
this.logger.info(meta.summalyProxy
? `(Proxy) Getting preview of ${url}@${lang} ...`
: `Getting preview of ${url}@${lang} ...`);
-
try {
- const summary = meta.summalyProxy ? await this.httpRequestService.getJson(`${meta.summalyProxy}?${query({
+ const summary = meta.summalyProxy ? await this.httpRequestService.getJson<ReturnType<typeof summaly.default>>(`${meta.summalyProxy}?${query({
url: url,
lang: lang ?? 'ja-JP',
})}`) : await summaly.default(url, {
diff --git a/yarn.lock b/yarn.lock
index 2e45bc1440..cdf2a19e1d 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -4272,6 +4272,7 @@ __metadata:
typeorm: 0.3.11
typescript: 4.9.4
ulid: 2.3.0
+ undici: ^5.14.0
unzipper: 0.10.11
uuid: 9.0.0
vary: 1.1.2
@@ -16665,6 +16666,15 @@ __metadata:
languageName: node
linkType: hard
+"undici@npm:^5.14.0, undici@npm:^5.5.1":
+ version: 5.14.0
+ resolution: "undici@npm:5.14.0"
+ dependencies:
+ busboy: ^1.6.0
+ checksum: 7a076e44d84b25844b4eb657034437b8b9bb91f17d347de474fdea1d4263ce7ae9406db79cd30de5642519277b4893f43073258bcc8fed420b295da3fdd11b26
+ languageName: node
+ linkType: hard
+
"undici@npm:^5.2.0":
version: 5.13.0
resolution: "undici@npm:5.13.0"
@@ -16674,15 +16684,6 @@ __metadata:
languageName: node
linkType: hard
-"undici@npm:^5.5.1":
- version: 5.14.0
- resolution: "undici@npm:5.14.0"
- dependencies:
- busboy: ^1.6.0
- checksum: 7a076e44d84b25844b4eb657034437b8b9bb91f17d347de474fdea1d4263ce7ae9406db79cd30de5642519277b4893f43073258bcc8fed420b295da3fdd11b26
- languageName: node
- linkType: hard
-
"union-value@npm:^1.0.0":
version: 1.0.1
resolution: "union-value@npm:1.0.1"