prometheus #20
15 changed files with 257 additions and 225 deletions
|
@ -24,8 +24,8 @@
|
||||||
"build": "pnpm build-pre && pnpm -r build && pnpm build-assets",
|
"build": "pnpm build-pre && pnpm -r build && pnpm build-assets",
|
||||||
"build-storybook": "pnpm --filter frontend build-storybook",
|
"build-storybook": "pnpm --filter frontend build-storybook",
|
||||||
"build-misskey-js-with-types": "pnpm build-pre && pnpm --filter backend... --filter=!misskey-js build && pnpm --filter backend generate-api-json --no-build && ncp packages/backend/built/api.json packages/misskey-js/generator/api.json && pnpm --filter misskey-js update-autogen-code && pnpm --filter misskey-js build && pnpm --filter misskey-js api",
|
"build-misskey-js-with-types": "pnpm build-pre && pnpm --filter backend... --filter=!misskey-js build && pnpm --filter backend generate-api-json --no-build && ncp packages/backend/built/api.json packages/misskey-js/generator/api.json && pnpm --filter misskey-js update-autogen-code && pnpm --filter misskey-js build && pnpm --filter misskey-js api",
|
||||||
"start": "pnpm check:connect && cd packages/backend && node ./built/boot/entry.js",
|
"start": "pnpm check:connect && cd packages/backend && cross-env RUN_MODE=web node ./built/boot/entry.js",
|
||||||
"start:test": "cd packages/backend && cross-env NODE_ENV=test node ./built/boot/entry.js",
|
"start:test": "cd packages/backend && cross-env NODE_ENV=test RUN_MODE=test node ./built/boot/entry.js",
|
||||||
"init": "pnpm migrate",
|
"init": "pnpm migrate",
|
||||||
"migrate": "cd packages/backend && pnpm migrate",
|
"migrate": "cd packages/backend && pnpm migrate",
|
||||||
"revert": "cd packages/backend && pnpm revert",
|
"revert": "cd packages/backend && pnpm revert",
|
||||||
|
@ -61,14 +61,14 @@
|
||||||
"terser": "5.36.0",
|
"terser": "5.36.0",
|
||||||
"typescript": "5.6.3",
|
"typescript": "5.6.3",
|
||||||
"esbuild": "0.24.0",
|
"esbuild": "0.24.0",
|
||||||
"glob": "11.0.0"
|
"glob": "11.0.0",
|
||||||
|
"cross-env": "7.0.3"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@misskey-dev/eslint-plugin": "2.0.3",
|
"@misskey-dev/eslint-plugin": "2.0.3",
|
||||||
"@types/node": "22.9.0",
|
"@types/node": "22.9.0",
|
||||||
"@typescript-eslint/eslint-plugin": "7.17.0",
|
"@typescript-eslint/eslint-plugin": "7.17.0",
|
||||||
"@typescript-eslint/parser": "7.17.0",
|
"@typescript-eslint/parser": "7.17.0",
|
||||||
"cross-env": "7.0.3",
|
|
||||||
"cypress": "13.15.2",
|
"cypress": "13.15.2",
|
||||||
"eslint": "9.14.0",
|
"eslint": "9.14.0",
|
||||||
"globals": "15.12.0",
|
"globals": "15.12.0",
|
||||||
|
|
|
@ -7,11 +7,6 @@ import Redis from 'ioredis';
|
||||||
import { loadConfig } from '../built/config.js';
|
import { loadConfig } from '../built/config.js';
|
||||||
import { createPostgresDataSource } from '../built/postgres.js';
|
import { createPostgresDataSource } from '../built/postgres.js';
|
||||||
|
|
||||||
const timeout = setTimeout(() => {
|
|
||||||
console.error('Timeout while connecting to databases.');
|
|
||||||
process.exit(1);
|
|
||||||
}, 120000);
|
|
||||||
|
|
||||||
const config = loadConfig();
|
const config = loadConfig();
|
||||||
|
|
||||||
async function connectToPostgres() {
|
async function connectToPostgres() {
|
||||||
|
@ -27,6 +22,7 @@ async function connectToRedis(redisOptions) {
|
||||||
lazyConnect: true,
|
lazyConnect: true,
|
||||||
reconnectOnError: false,
|
reconnectOnError: false,
|
||||||
showFriendlyErrorStack: true,
|
showFriendlyErrorStack: true,
|
||||||
|
connectTimeout: 10000,
|
||||||
});
|
});
|
||||||
redis.on('error', e => reject(e));
|
redis.on('error', e => reject(e));
|
||||||
|
|
||||||
|
@ -60,4 +56,4 @@ const promises = Array
|
||||||
|
|
||||||
await Promise.allSettled(promises);
|
await Promise.allSettled(promises);
|
||||||
|
|
||||||
clearTimeout(timeout);
|
process.exit(0);
|
||||||
|
|
|
@ -20,6 +20,7 @@ import type { Config } from '@/config.js';
|
||||||
import { showMachineInfo } from '@/misc/show-machine-info.js';
|
import { showMachineInfo } from '@/misc/show-machine-info.js';
|
||||||
import { envOption } from '@/env.js';
|
import { envOption } from '@/env.js';
|
||||||
import { jobQueue, server } from './common.js';
|
import { jobQueue, server } from './common.js';
|
||||||
|
import { metricGauge } from '@/server/api/MetricsService.js';
|
||||||
|
|
||||||
const _filename = fileURLToPath(import.meta.url);
|
const _filename = fileURLToPath(import.meta.url);
|
||||||
const _dirname = dirname(_filename);
|
const _dirname = dirname(_filename);
|
||||||
|
@ -31,18 +32,24 @@ const bootLogger = logger.createSubLogger('boot', 'magenta');
|
||||||
|
|
||||||
const themeColor = chalk.hex('#86b300');
|
const themeColor = chalk.hex('#86b300');
|
||||||
|
|
||||||
const mBuildInfo = new prom.Gauge({
|
const mBuildInfo = metricGauge({
|
||||||
name: 'misskey_build_info',
|
name: 'misskey_build_info',
|
||||||
help: 'Misskey build information',
|
help: 'Misskey build information',
|
||||||
labelNames: ['gitCommit', 'gitDescribe', 'node_version']
|
labelNames: ['gitCommit', 'gitDescribe', 'node_version']
|
||||||
});
|
});
|
||||||
|
|
||||||
mBuildInfo.set({
|
mBuildInfo?.set({
|
||||||
gitCommit: meta.gitCommit || 'unknown',
|
gitCommit: meta.gitCommit || 'unknown',
|
||||||
gitDescribe: meta.gitDescribe || 'unknown',
|
gitDescribe: meta.gitDescribe || 'unknown',
|
||||||
node_version: process.version
|
node_version: process.version
|
||||||
}, 1);
|
}, 1);
|
||||||
|
|
||||||
|
const mStartupTime = metricGauge({
|
||||||
|
name: 'misskey_startup_time',
|
||||||
|
help: 'Misskey startup time',
|
||||||
|
labelNames: ['pid']
|
||||||
|
});
|
||||||
|
|
||||||
function greet() {
|
function greet() {
|
||||||
if (!envOption.quiet) {
|
if (!envOption.quiet) {
|
||||||
//#region Misskey logo
|
//#region Misskey logo
|
||||||
|
@ -112,6 +119,8 @@ export async function masterMain() {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
mStartupTime?.set({ pid: process.pid }, Date.now());
|
||||||
|
|
||||||
if (envOption.disableClustering) {
|
if (envOption.disableClustering) {
|
||||||
if (envOption.onlyServer) {
|
if (envOption.onlyServer) {
|
||||||
await server();
|
await server();
|
||||||
|
|
|
@ -31,15 +31,17 @@ export type UserWebhookDeliverQueue = Bull.Queue<UserWebhookDeliverJobData>;
|
||||||
export type SystemWebhookDeliverQueue = Bull.Queue<SystemWebhookDeliverJobData>;
|
export type SystemWebhookDeliverQueue = Bull.Queue<SystemWebhookDeliverJobData>;
|
||||||
|
|
||||||
function withMetrics<T>(queue: Bull.Queue<T>): Bull.Queue<T> {
|
function withMetrics<T>(queue: Bull.Queue<T>): Bull.Queue<T> {
|
||||||
|
if (process.env.NODE_ENV !== 'test') {
|
||||||
setInterval(async () => {
|
setInterval(async () => {
|
||||||
mActiveJobs.set({ queue: queue.name }, await queue.getActiveCount());
|
mActiveJobs?.set({ queue: queue.name }, await queue.getActiveCount());
|
||||||
mDelayedJobs.set({ queue: queue.name }, await queue.getDelayedCount());
|
mDelayedJobs?.set({ queue: queue.name }, await queue.getDelayedCount());
|
||||||
mWaitingJobs.set({ queue: queue.name }, await queue.getWaitingCount());
|
mWaitingJobs?.set({ queue: queue.name }, await queue.getWaitingCount());
|
||||||
}, 2000);
|
}, 2000);
|
||||||
|
|
||||||
queue.on('waiting', () => {
|
queue.on('waiting', () => {
|
||||||
mJobReceivedCounter.inc({ queue: queue.name });
|
mJobReceivedCounter?.inc({ queue: queue.name });
|
||||||
});
|
});
|
||||||
|
}
|
||||||
|
|
||||||
return queue;
|
return queue;
|
||||||
}
|
}
|
||||||
|
|
|
@ -39,17 +39,18 @@ import { ApPersonService } from './models/ApPersonService.js';
|
||||||
import { ApQuestionService } from './models/ApQuestionService.js';
|
import { ApQuestionService } from './models/ApQuestionService.js';
|
||||||
import type { Resolver } from './ApResolverService.js';
|
import type { Resolver } from './ApResolverService.js';
|
||||||
import type { IAccept, IAdd, IAnnounce, IBlock, ICreate, IDelete, IFlag, IFollow, ILike, IObject, IReject, IRemove, IUndo, IUpdate, IMove, IPost } from './type.js';
|
import type { IAccept, IAdd, IAnnounce, IBlock, ICreate, IDelete, IFlag, IFollow, ILike, IObject, IReject, IRemove, IUndo, IUpdate, IMove, IPost } from './type.js';
|
||||||
|
import { metricCounter } from '@/server/api/MetricsService.js';
|
||||||
|
|
||||||
|
const mInboxReceived = metricCounter({
|
||||||
|
name: 'misskey_ap_inbox_received_total',
|
||||||
|
help: 'Total number of activities received by AP inbox',
|
||||||
|
labelNames: ['host', 'type'],
|
||||||
|
});
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class ApInboxService {
|
export class ApInboxService {
|
||||||
private logger: Logger;
|
private logger: Logger;
|
||||||
|
|
||||||
private mInboxReceived = new prom.Counter({
|
|
||||||
name: 'misskey_ap_inbox_received_total',
|
|
||||||
help: 'Total number of activities received by AP inbox',
|
|
||||||
labelNames: ['host', 'type'],
|
|
||||||
});
|
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
@Inject(DI.config)
|
@Inject(DI.config)
|
||||||
private config: Config,
|
private config: Config,
|
||||||
|
@ -138,49 +139,49 @@ export class ApInboxService {
|
||||||
if (actor.isSuspended) return;
|
if (actor.isSuspended) return;
|
||||||
|
|
||||||
if (isCreate(activity)) {
|
if (isCreate(activity)) {
|
||||||
this.mInboxReceived.inc({ host: actor.host, type: 'create' });
|
mInboxReceived?.inc({ host: actor.host, type: 'create' });
|
||||||
return await this.create(actor, activity);
|
return await this.create(actor, activity);
|
||||||
} else if (isDelete(activity)) {
|
} else if (isDelete(activity)) {
|
||||||
this.mInboxReceived.inc({ host: actor.host, type: 'delete' });
|
mInboxReceived?.inc({ host: actor.host, type: 'delete' });
|
||||||
return await this.delete(actor, activity);
|
return await this.delete(actor, activity);
|
||||||
} else if (isUpdate(activity)) {
|
} else if (isUpdate(activity)) {
|
||||||
this.mInboxReceived.inc({ host: actor.host, type: 'update' });
|
mInboxReceived?.inc({ host: actor.host, type: 'update' });
|
||||||
return await this.update(actor, activity);
|
return await this.update(actor, activity);
|
||||||
} else if (isFollow(activity)) {
|
} else if (isFollow(activity)) {
|
||||||
this.mInboxReceived.inc({ host: actor.host, type: 'follow' });
|
mInboxReceived?.inc({ host: actor.host, type: 'follow' });
|
||||||
return await this.follow(actor, activity);
|
return await this.follow(actor, activity);
|
||||||
} else if (isAccept(activity)) {
|
} else if (isAccept(activity)) {
|
||||||
this.mInboxReceived.inc({ host: actor.host, type: 'accept' });
|
mInboxReceived?.inc({ host: actor.host, type: 'accept' });
|
||||||
return await this.accept(actor, activity);
|
return await this.accept(actor, activity);
|
||||||
} else if (isReject(activity)) {
|
} else if (isReject(activity)) {
|
||||||
this.mInboxReceived.inc({ host: actor.host, type: 'reject' });
|
mInboxReceived?.inc({ host: actor.host, type: 'reject' });
|
||||||
return await this.reject(actor, activity);
|
return await this.reject(actor, activity);
|
||||||
} else if (isAdd(activity)) {
|
} else if (isAdd(activity)) {
|
||||||
this.mInboxReceived.inc({ host: actor.host, type: 'add' });
|
mInboxReceived?.inc({ host: actor.host, type: 'add' });
|
||||||
return await this.add(actor, activity);
|
return await this.add(actor, activity);
|
||||||
} else if (isRemove(activity)) {
|
} else if (isRemove(activity)) {
|
||||||
this.mInboxReceived.inc({ host: actor.host, type: 'remove' });
|
mInboxReceived?.inc({ host: actor.host, type: 'remove' });
|
||||||
return await this.remove(actor, activity);
|
return await this.remove(actor, activity);
|
||||||
} else if (isAnnounce(activity)) {
|
} else if (isAnnounce(activity)) {
|
||||||
this.mInboxReceived.inc({ host: actor.host, type: 'announce' });
|
mInboxReceived?.inc({ host: actor.host, type: 'announce' });
|
||||||
return await this.announce(actor, activity);
|
return await this.announce(actor, activity);
|
||||||
} else if (isLike(activity)) {
|
} else if (isLike(activity)) {
|
||||||
this.mInboxReceived.inc({ host: actor.host, type: 'like' });
|
mInboxReceived?.inc({ host: actor.host, type: 'like' });
|
||||||
return await this.like(actor, activity);
|
return await this.like(actor, activity);
|
||||||
} else if (isUndo(activity)) {
|
} else if (isUndo(activity)) {
|
||||||
this.mInboxReceived.inc({ host: actor.host, type: 'undo' });
|
mInboxReceived?.inc({ host: actor.host, type: 'undo' });
|
||||||
return await this.undo(actor, activity);
|
return await this.undo(actor, activity);
|
||||||
} else if (isBlock(activity)) {
|
} else if (isBlock(activity)) {
|
||||||
this.mInboxReceived.inc({ host: actor.host, type: 'block' });
|
mInboxReceived?.inc({ host: actor.host, type: 'block' });
|
||||||
return await this.block(actor, activity);
|
return await this.block(actor, activity);
|
||||||
} else if (isFlag(activity)) {
|
} else if (isFlag(activity)) {
|
||||||
this.mInboxReceived.inc({ host: actor.host, type: 'flag' });
|
mInboxReceived?.inc({ host: actor.host, type: 'flag' });
|
||||||
return await this.flag(actor, activity);
|
return await this.flag(actor, activity);
|
||||||
} else if (isMove(activity)) {
|
} else if (isMove(activity)) {
|
||||||
this.mInboxReceived.inc({ host: actor.host, type: 'move' });
|
mInboxReceived?.inc({ host: actor.host, type: 'move' });
|
||||||
return await this.move(actor, activity);
|
return await this.move(actor, activity);
|
||||||
} else {
|
} else {
|
||||||
this.mInboxReceived.inc({ host: actor.host, type: 'unknown' });
|
mInboxReceived?.inc({ host: actor.host, type: 'unknown' });
|
||||||
return `unrecognized activity type: ${activity.type}`;
|
return `unrecognized activity type: ${activity.type}`;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -491,9 +492,9 @@ export class ApInboxService {
|
||||||
formerType = 'Note';
|
formerType = 'Note';
|
||||||
}
|
}
|
||||||
|
|
||||||
if (validPost.includes(formerType)) {
|
if (validPost?.includes(formerType)) {
|
||||||
return await this.deleteNote(actor, uri);
|
return await this.deleteNote(actor, uri);
|
||||||
} else if (validActor.includes(formerType)) {
|
} else if (validActor?.includes(formerType)) {
|
||||||
return await this.deleteActor(actor, uri);
|
return await this.deleteActor(actor, uri);
|
||||||
} else {
|
} else {
|
||||||
return `Unknown type ${formerType}`;
|
return `Unknown type ${formerType}`;
|
||||||
|
|
|
@ -7,7 +7,6 @@
|
||||||
import pg from 'pg';
|
import pg from 'pg';
|
||||||
import { DataSource, Logger } from 'typeorm';
|
import { DataSource, Logger } from 'typeorm';
|
||||||
import * as highlight from 'cli-highlight';
|
import * as highlight from 'cli-highlight';
|
||||||
import * as prom from 'prom-client';
|
|
||||||
import { createHash } from 'crypto';
|
import { createHash } from 'crypto';
|
||||||
import { entities as charts } from '@/core/chart/entities.js';
|
import { entities as charts } from '@/core/chart/entities.js';
|
||||||
|
|
||||||
|
@ -85,6 +84,7 @@ import { Config } from '@/config.js';
|
||||||
import MisskeyLogger from '@/logger.js';
|
import MisskeyLogger from '@/logger.js';
|
||||||
import { bindThis } from '@/decorators.js';
|
import { bindThis } from '@/decorators.js';
|
||||||
import { MemoryKVCache } from './misc/cache.js';
|
import { MemoryKVCache } from './misc/cache.js';
|
||||||
|
import { metricCounter, metricHistogram } from './server/api/MetricsService.js';
|
||||||
|
|
||||||
pg.types.setTypeParser(20, Number);
|
pg.types.setTypeParser(20, Number);
|
||||||
|
|
||||||
|
@ -125,19 +125,19 @@ function extractQueryTags(query: string): QueryTagCache {
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
const mQueryCounter = new prom.Counter({
|
const mQueryCounter = metricCounter({
|
||||||
name: 'misskey_postgres_query_total',
|
name: 'misskey_postgres_query_total',
|
||||||
help: 'Total queries to postgres',
|
help: 'Total queries to postgres',
|
||||||
labelNames: ['join', 'from', 'hash'],
|
labelNames: ['join', 'from', 'hash'],
|
||||||
});
|
});
|
||||||
|
|
||||||
const mQueryErrorCounter = new prom.Counter({
|
const mQueryErrorCounter = metricCounter({
|
||||||
name: 'misskey_postgres_query_error_total',
|
name: 'misskey_postgres_query_error_total',
|
||||||
help: 'Total errors in queries to postgres',
|
help: 'Total errors in queries to postgres',
|
||||||
labelNames: ['join', 'from', 'hash'],
|
labelNames: ['join', 'from', 'hash'],
|
||||||
});
|
});
|
||||||
|
|
||||||
const mSlowQueryHisto = new prom.Histogram({
|
const mSlowQueryHisto = metricHistogram({
|
||||||
name: 'misskey_postgres_query_slow_duration_seconds',
|
name: 'misskey_postgres_query_slow_duration_seconds',
|
||||||
help: 'Duration of slow queries to postgres',
|
help: 'Duration of slow queries to postgres',
|
||||||
labelNames: ['join', 'from', 'hash'],
|
labelNames: ['join', 'from', 'hash'],
|
||||||
|
@ -170,7 +170,7 @@ class MyCustomLogger implements Logger {
|
||||||
|
|
||||||
@bindThis
|
@bindThis
|
||||||
public logQuery(query: string, parameters?: any[]) {
|
public logQuery(query: string, parameters?: any[]) {
|
||||||
mQueryCounter.inc(this.getQueryTags(query));
|
mQueryCounter?.inc(this.getQueryTags(query));
|
||||||
|
|
||||||
if (this.metricOnly) {
|
if (this.metricOnly) {
|
||||||
return;
|
return;
|
||||||
|
@ -180,7 +180,7 @@ class MyCustomLogger implements Logger {
|
||||||
|
|
||||||
@bindThis
|
@bindThis
|
||||||
public logQueryError(error: string, query: string, parameters?: any[]) {
|
public logQueryError(error: string, query: string, parameters?: any[]) {
|
||||||
mQueryErrorCounter.inc(this.getQueryTags(query));
|
mQueryErrorCounter?.inc(this.getQueryTags(query));
|
||||||
|
|
||||||
if (this.metricOnly) {
|
if (this.metricOnly) {
|
||||||
return;
|
return;
|
||||||
|
@ -190,7 +190,7 @@ class MyCustomLogger implements Logger {
|
||||||
|
|
||||||
@bindThis
|
@bindThis
|
||||||
public logQuerySlow(time: number, query: string, parameters?: any[]) {
|
public logQuerySlow(time: number, query: string, parameters?: any[]) {
|
||||||
mSlowQueryHisto.observe(this.getQueryTags(query), time);
|
mSlowQueryHisto?.observe(this.getQueryTags(query), time);
|
||||||
|
|
||||||
if (this.metricOnly) {
|
if (this.metricOnly) {
|
||||||
return;
|
return;
|
||||||
|
@ -341,8 +341,8 @@ export function createPostgresDataSource(config: Config, isMain = false) {
|
||||||
db: config.redis.db ?? 0,
|
db: config.redis.db ?? 0,
|
||||||
},
|
},
|
||||||
} : false,
|
} : false,
|
||||||
logging: log ? 'all' : ['query'],
|
logging: true,
|
||||||
logger: (isMain || log) ? new MyCustomLogger(!log) : undefined,
|
logger: new MyCustomLogger(!log),
|
||||||
maxQueryExecutionTime: 500,
|
maxQueryExecutionTime: 500,
|
||||||
entities: entities,
|
entities: entities,
|
||||||
migrations: ['../../migration/*.js'],
|
migrations: ['../../migration/*.js'],
|
||||||
|
|
|
@ -196,7 +196,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
|
||||||
})
|
})
|
||||||
.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
|
.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
|
||||||
.on('stalled', (jobId) => {
|
.on('stalled', (jobId) => {
|
||||||
mStalledWorkerCounter.inc({ queue: QUEUE.SYSTEM });
|
mStalledWorkerCounter?.inc({ queue: QUEUE.SYSTEM });
|
||||||
logger.warn(`stalled id=${jobId}`);
|
logger.warn(`stalled id=${jobId}`);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
@ -256,7 +256,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
|
||||||
})
|
})
|
||||||
.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
|
.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
|
||||||
.on('stalled', (jobId) => {
|
.on('stalled', (jobId) => {
|
||||||
mStalledWorkerCounter.inc({ queue: QUEUE.DB });
|
mStalledWorkerCounter?.inc({ queue: QUEUE.DB });
|
||||||
logger.warn(`stalled id=${jobId}`);
|
logger.warn(`stalled id=${jobId}`);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
@ -299,7 +299,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
|
||||||
})
|
})
|
||||||
.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
|
.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
|
||||||
.on('stalled', (jobId) => {
|
.on('stalled', (jobId) => {
|
||||||
mStalledWorkerCounter.inc({ queue: QUEUE.DELIVER });
|
mStalledWorkerCounter?.inc({ queue: QUEUE.DELIVER });
|
||||||
logger.warn(`stalled id=${jobId}`);
|
logger.warn(`stalled id=${jobId}`);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
@ -342,7 +342,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
|
||||||
})
|
})
|
||||||
.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
|
.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
|
||||||
.on('stalled', (jobId) => {
|
.on('stalled', (jobId) => {
|
||||||
mStalledWorkerCounter.inc({ queue: QUEUE.INBOX });
|
mStalledWorkerCounter?.inc({ queue: QUEUE.INBOX });
|
||||||
logger.warn(`stalled id=${jobId}`);
|
logger.warn(`stalled id=${jobId}`);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
@ -385,7 +385,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
|
||||||
})
|
})
|
||||||
.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
|
.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
|
||||||
.on('stalled', (jobId) => {
|
.on('stalled', (jobId) => {
|
||||||
mStalledWorkerCounter.inc({ queue: QUEUE.USER_WEBHOOK_DELIVER });
|
mStalledWorkerCounter?.inc({ queue: QUEUE.USER_WEBHOOK_DELIVER });
|
||||||
logger.warn(`stalled id=${jobId}`);
|
logger.warn(`stalled id=${jobId}`);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
@ -428,7 +428,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
|
||||||
})
|
})
|
||||||
.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
|
.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
|
||||||
.on('stalled', (jobId) => {
|
.on('stalled', (jobId) => {
|
||||||
mStalledWorkerCounter.inc({ queue: QUEUE.SYSTEM_WEBHOOK_DELIVER });
|
mStalledWorkerCounter?.inc({ queue: QUEUE.SYSTEM_WEBHOOK_DELIVER });
|
||||||
logger.warn(`stalled id=${jobId}`);
|
logger.warn(`stalled id=${jobId}`);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
@ -478,7 +478,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
|
||||||
})
|
})
|
||||||
.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
|
.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
|
||||||
.on('stalled', (jobId) => {
|
.on('stalled', (jobId) => {
|
||||||
mStalledWorkerCounter.inc({ queue: QUEUE.RELATIONSHIP });
|
mStalledWorkerCounter?.inc({ queue: QUEUE.RELATIONSHIP });
|
||||||
logger.warn(`stalled id=${jobId}`);
|
logger.warn(`stalled id=${jobId}`);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
@ -522,7 +522,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
|
||||||
})
|
})
|
||||||
.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
|
.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
|
||||||
.on('stalled', (jobId) => {
|
.on('stalled', (jobId) => {
|
||||||
mStalledWorkerCounter.inc({ queue: QUEUE.OBJECT_STORAGE });
|
mStalledWorkerCounter?.inc({ queue: QUEUE.OBJECT_STORAGE });
|
||||||
logger.warn(`stalled id=${jobId}`);
|
logger.warn(`stalled id=${jobId}`);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,30 +1,30 @@
|
||||||
import * as prom from 'prom-client';
|
import { metricCounter, metricGauge } from '@/server/api/MetricsService.js';
|
||||||
|
|
||||||
export const mJobReceivedCounter = new prom.Counter({
|
export const mJobReceivedCounter = metricCounter({
|
||||||
name: 'misskey_queue_jobs_received_total',
|
name: 'misskey_queue_jobs_received_total',
|
||||||
help: 'Total number of jobs received by queue',
|
help: 'Total number of jobs received by queue',
|
||||||
labelNames: ['queue'],
|
labelNames: ['queue'],
|
||||||
});
|
});
|
||||||
|
|
||||||
export const mActiveJobs = new prom.Gauge({
|
export const mActiveJobs = metricGauge({
|
||||||
name: 'misskey_queue_active_jobs',
|
name: 'misskey_queue_active_jobs',
|
||||||
help: 'Number of active jobs in queue',
|
help: 'Number of active jobs in queue',
|
||||||
labelNames: ['queue'],
|
labelNames: ['queue'],
|
||||||
});
|
});
|
||||||
|
|
||||||
export const mDelayedJobs = new prom.Gauge({
|
export const mDelayedJobs = metricGauge({
|
||||||
name: 'misskey_queue_delayed_jobs',
|
name: 'misskey_queue_delayed_jobs',
|
||||||
help: 'Number of delayed jobs in queue',
|
help: 'Number of delayed jobs in queue',
|
||||||
labelNames: ['queue'],
|
labelNames: ['queue'],
|
||||||
});
|
});
|
||||||
|
|
||||||
export const mWaitingJobs = new prom.Gauge({
|
export const mWaitingJobs = metricGauge({
|
||||||
name: 'misskey_queue_waiting_jobs',
|
name: 'misskey_queue_waiting_jobs',
|
||||||
help: 'Number of waiting jobs in queue',
|
help: 'Number of waiting jobs in queue',
|
||||||
labelNames: ['queue'],
|
labelNames: ['queue'],
|
||||||
});
|
});
|
||||||
|
|
||||||
export const mStalledWorkerCounter = new prom.Counter({
|
export const mStalledWorkerCounter = metricCounter({
|
||||||
name: 'misskey_queue_stalled_workers_total',
|
name: 'misskey_queue_stalled_workers_total',
|
||||||
help: 'Total number of stalled workers',
|
help: 'Total number of stalled workers',
|
||||||
labelNames: ['queue'],
|
labelNames: ['queue'],
|
||||||
|
|
|
@ -32,47 +32,48 @@ import { MiMeta } from '@/models/Meta.js';
|
||||||
import { DI } from '@/di-symbols.js';
|
import { DI } from '@/di-symbols.js';
|
||||||
import { QueueLoggerService } from '../QueueLoggerService.js';
|
import { QueueLoggerService } from '../QueueLoggerService.js';
|
||||||
import type { InboxJobData } from '../types.js';
|
import type { InboxJobData } from '../types.js';
|
||||||
|
import { metricCounter, metricHistogram } from '@/server/api/MetricsService.js';
|
||||||
|
|
||||||
type UpdateInstanceJob = {
|
type UpdateInstanceJob = {
|
||||||
latestRequestReceivedAt: Date,
|
latestRequestReceivedAt: Date,
|
||||||
shouldUnsuspend: boolean,
|
shouldUnsuspend: boolean,
|
||||||
};
|
};
|
||||||
|
|
||||||
@Injectable()
|
const mIncomingApProcessingTime = metricHistogram({
|
||||||
export class InboxProcessorService implements OnApplicationShutdown {
|
|
||||||
private logger: Logger;
|
|
||||||
private updateInstanceQueue: CollapsedQueue<MiNote['id'], UpdateInstanceJob>;
|
|
||||||
|
|
||||||
private mIncomingApProcessingTime = new prom.Histogram({
|
|
||||||
name: 'misskey_incoming_ap_processing_time',
|
name: 'misskey_incoming_ap_processing_time',
|
||||||
help: 'Incoming AP processing time in seconds',
|
help: 'Incoming AP processing time in seconds',
|
||||||
labelNames: ['incoming_host', 'incoming_type', 'success'],
|
labelNames: ['incoming_host', 'incoming_type', 'success'],
|
||||||
buckets: [0.01, 0.1, 0.5, 1, 5, 10, 30, 60, 300, 1800],
|
buckets: [0.01, 0.1, 0.5, 1, 5, 10, 30, 60, 300, 1800],
|
||||||
});
|
});
|
||||||
|
|
||||||
private mIncomingApEvent = new prom.Counter({
|
const mIncomingApEvent = metricCounter({
|
||||||
name: 'misskey_incoming_ap_event',
|
name: 'misskey_incoming_ap_event',
|
||||||
help: 'Incoming AP event',
|
help: 'Incoming AP event',
|
||||||
labelNames: ['incoming_host', 'incoming_type'],
|
labelNames: ['incoming_host', 'incoming_type'],
|
||||||
});
|
});
|
||||||
|
|
||||||
private mIncomingApEventAccepted = new prom.Counter({
|
const mIncomingApEventAccepted = metricCounter({
|
||||||
name: 'misskey_incoming_ap_event_accepted',
|
name: 'misskey_incoming_ap_event_accepted',
|
||||||
help: 'Incoming AP event accepted',
|
help: 'Incoming AP event accepted',
|
||||||
labelNames: ['incoming_host', 'incoming_type'],
|
labelNames: ['incoming_host', 'incoming_type'],
|
||||||
});
|
});
|
||||||
|
|
||||||
private mIncomingApReject = new prom.Counter({
|
const mIncomingApReject = metricCounter({
|
||||||
name: 'misskey_incoming_ap_reject',
|
name: 'misskey_incoming_ap_reject',
|
||||||
help: 'Incoming AP reject',
|
help: 'Incoming AP reject',
|
||||||
labelNames: ['incoming_host', 'incoming_type', 'reason'],
|
labelNames: ['incoming_host', 'incoming_type', 'reason'],
|
||||||
});
|
});
|
||||||
|
|
||||||
private mincomingApProcessingError = new prom.Counter({
|
const mincomingApProcessingError = metricCounter({
|
||||||
name: 'misskey_incoming_ap_processing_error',
|
name: 'misskey_incoming_ap_processing_error',
|
||||||
help: 'Incoming AP processing error',
|
help: 'Incoming AP processing error',
|
||||||
labelNames: ['incoming_host', 'incoming_type'],
|
labelNames: ['incoming_host', 'incoming_type'],
|
||||||
});
|
});
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class InboxProcessorService implements OnApplicationShutdown {
|
||||||
|
private logger: Logger;
|
||||||
|
private updateInstanceQueue: CollapsedQueue<MiNote['id'], UpdateInstanceJob>;
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
@Inject(DI.meta)
|
@Inject(DI.meta)
|
||||||
|
@ -106,34 +107,34 @@ export class InboxProcessorService implements OnApplicationShutdown {
|
||||||
|
|
||||||
const host = this.utilityService.toPuny(new URL(signature.keyId).hostname);
|
const host = this.utilityService.toPuny(new URL(signature.keyId).hostname);
|
||||||
|
|
||||||
const incCounter = <T extends 'incoming_host' | 'incoming_type', U>(counter: prom.Counter<T>, addn_labels: U) => {
|
const incCounter = <T extends 'incoming_host' | 'incoming_type', U>(counter: prom.Counter<T> | null, addn_labels: U) => {
|
||||||
if (Array.isArray(activity.type)) {
|
if (Array.isArray(activity.type)) {
|
||||||
for (const t of activity.type) {
|
for (const t of activity.type) {
|
||||||
counter.inc({ incoming_host: host.toString(), incoming_type: t, ...addn_labels });
|
counter?.inc({ incoming_host: host.toString(), incoming_type: t, ...addn_labels });
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
counter.inc({ incoming_host: host.toString(), incoming_type: activity.type ?? 'unknown', ...addn_labels });
|
counter?.inc({ incoming_host: host.toString(), incoming_type: activity.type ?? 'unknown', ...addn_labels });
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const observeHistogram = <T extends 'incoming_host' | 'incoming_type', U>(histogram: prom.Histogram<T>, addn_labels: U, value: number) => {
|
const observeHistogram = <T extends 'incoming_host' | 'incoming_type', U>(histogram: prom.Histogram<T> | null, addn_labels: U, value: number) => {
|
||||||
if (Array.isArray(activity.type)) {
|
if (Array.isArray(activity.type)) {
|
||||||
for (const t of activity.type) {
|
for (const t of activity.type) {
|
||||||
histogram.observe({ incoming_host: host.toString(), incoming_type: t, ...addn_labels }, value);
|
histogram?.observe({ incoming_host: host.toString(), incoming_type: t, ...addn_labels }, value);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
histogram.observe({ incoming_host: host.toString(), incoming_type: activity.type ?? 'unknown', ...addn_labels }, value);
|
histogram?.observe({ incoming_host: host.toString(), incoming_type: activity.type ?? 'unknown', ...addn_labels }, value);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
if (!this.utilityService.isFederationAllowedHost(host)) {
|
if (!this.utilityService.isFederationAllowedHost(host)) {
|
||||||
incCounter(this.mIncomingApReject, { reason: 'host_not_allowed' });
|
incCounter(mIncomingApReject, { reason: 'host_not_allowed' });
|
||||||
return `Blocked request: ${host}`;
|
return `Blocked request: ${host}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
const keyIdLower = signature.keyId.toLowerCase();
|
const keyIdLower = signature.keyId.toLowerCase();
|
||||||
if (keyIdLower.startsWith('acct:')) {
|
if (keyIdLower.startsWith('acct:')) {
|
||||||
incCounter(this.mIncomingApReject, { reason: 'keyid_acct' });
|
incCounter(mIncomingApReject, { reason: 'keyid_acct' });
|
||||||
return `Old keyId is no longer supported. ${keyIdLower}`;
|
return `Old keyId is no longer supported. ${keyIdLower}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -151,7 +152,7 @@ export class InboxProcessorService implements OnApplicationShutdown {
|
||||||
// 対象が4xxならスキップ
|
// 対象が4xxならスキップ
|
||||||
if (err instanceof StatusError) {
|
if (err instanceof StatusError) {
|
||||||
if (!err.isRetryable) {
|
if (!err.isRetryable) {
|
||||||
incCounter(this.mIncomingApReject, { reason: 'actor_key_unresolvable' });
|
incCounter(mIncomingApReject, { reason: 'actor_key_unresolvable' });
|
||||||
throw new Bull.UnrecoverableError(`skip: Ignored deleted actors on both ends ${activity.actor} - ${err.statusCode}`);
|
throw new Bull.UnrecoverableError(`skip: Ignored deleted actors on both ends ${activity.actor} - ${err.statusCode}`);
|
||||||
}
|
}
|
||||||
throw new Error(`Error in actor ${activity.actor} - ${err.statusCode}`);
|
throw new Error(`Error in actor ${activity.actor} - ${err.statusCode}`);
|
||||||
|
@ -161,13 +162,13 @@ export class InboxProcessorService implements OnApplicationShutdown {
|
||||||
|
|
||||||
// それでもわからなければ終了
|
// それでもわからなければ終了
|
||||||
if (authUser == null) {
|
if (authUser == null) {
|
||||||
incCounter(this.mIncomingApReject, { reason: 'actor_unresolvable' });
|
incCounter(mIncomingApReject, { reason: 'actor_unresolvable' });
|
||||||
throw new Bull.UnrecoverableError('skip: failed to resolve user');
|
throw new Bull.UnrecoverableError('skip: failed to resolve user');
|
||||||
}
|
}
|
||||||
|
|
||||||
// publicKey がなくても終了
|
// publicKey がなくても終了
|
||||||
if (authUser.key == null) {
|
if (authUser.key == null) {
|
||||||
incCounter(this.mIncomingApReject, { reason: 'publickey_unresolvable' });
|
incCounter(mIncomingApReject, { reason: 'publickey_unresolvable' });
|
||||||
throw new Bull.UnrecoverableError('skip: failed to resolve user publicKey');
|
throw new Bull.UnrecoverableError('skip: failed to resolve user publicKey');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -180,7 +181,7 @@ export class InboxProcessorService implements OnApplicationShutdown {
|
||||||
const ldSignature = activity.signature;
|
const ldSignature = activity.signature;
|
||||||
if (ldSignature) {
|
if (ldSignature) {
|
||||||
if (ldSignature.type !== 'RsaSignature2017') {
|
if (ldSignature.type !== 'RsaSignature2017') {
|
||||||
incCounter(this.mIncomingApReject, { reason: 'ld_signature_unsupported' });
|
incCounter(mIncomingApReject, { reason: 'ld_signature_unsupported' });
|
||||||
throw new Bull.UnrecoverableError(`skip: unsupported LD-signature type ${ldSignature.type}`);
|
throw new Bull.UnrecoverableError(`skip: unsupported LD-signature type ${ldSignature.type}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -194,12 +195,12 @@ export class InboxProcessorService implements OnApplicationShutdown {
|
||||||
// keyIdからLD-Signatureのユーザーを取得
|
// keyIdからLD-Signatureのユーザーを取得
|
||||||
authUser = await this.apDbResolverService.getAuthUserFromKeyId(ldSignature.creator);
|
authUser = await this.apDbResolverService.getAuthUserFromKeyId(ldSignature.creator);
|
||||||
if (authUser == null) {
|
if (authUser == null) {
|
||||||
incCounter(this.mIncomingApReject, { reason: 'ld_signature_user_unresolvable' });
|
incCounter(mIncomingApReject, { reason: 'ld_signature_user_unresolvable' });
|
||||||
throw new Bull.UnrecoverableError('skip: LD-Signatureのユーザーが取得できませんでした');
|
throw new Bull.UnrecoverableError('skip: LD-Signatureのユーザーが取得できませんでした');
|
||||||
}
|
}
|
||||||
|
|
||||||
if (authUser.key == null) {
|
if (authUser.key == null) {
|
||||||
incCounter(this.mIncomingApReject, { reason: 'ld_signature_publickey_unavailable' });
|
incCounter(mIncomingApReject, { reason: 'ld_signature_publickey_unavailable' });
|
||||||
throw new Bull.UnrecoverableError('skip: LD-SignatureのユーザーはpublicKeyを持っていませんでした');
|
throw new Bull.UnrecoverableError('skip: LD-SignatureのユーザーはpublicKeyを持っていませんでした');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -208,7 +209,7 @@ export class InboxProcessorService implements OnApplicationShutdown {
|
||||||
// LD-Signature検証
|
// LD-Signature検証
|
||||||
const verified = await jsonLd.verifyRsaSignature2017(activity, authUser.key.keyPem).catch(() => false);
|
const verified = await jsonLd.verifyRsaSignature2017(activity, authUser.key.keyPem).catch(() => false);
|
||||||
if (!verified) {
|
if (!verified) {
|
||||||
incCounter(this.mIncomingApReject, { reason: 'ld_signature_verification_failed' });
|
incCounter(mIncomingApReject, { reason: 'ld_signature_verification_failed' });
|
||||||
throw new Bull.UnrecoverableError('skip: LD-Signatureの検証に失敗しました');
|
throw new Bull.UnrecoverableError('skip: LD-Signatureの検証に失敗しました');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -231,17 +232,17 @@ export class InboxProcessorService implements OnApplicationShutdown {
|
||||||
|
|
||||||
// もう一度actorチェック
|
// もう一度actorチェック
|
||||||
if (authUser.user.uri !== activity.actor) {
|
if (authUser.user.uri !== activity.actor) {
|
||||||
incCounter(this.mIncomingApReject, { reason: 'ld_signature_actor_mismatch' });
|
incCounter(mIncomingApReject, { reason: 'ld_signature_actor_mismatch' });
|
||||||
throw new Bull.UnrecoverableError(`skip: LD-Signature user(${authUser.user.uri}) !== activity.actor(${activity.actor})`);
|
throw new Bull.UnrecoverableError(`skip: LD-Signature user(${authUser.user.uri}) !== activity.actor(${activity.actor})`);
|
||||||
}
|
}
|
||||||
|
|
||||||
const ldHost = this.utilityService.extractDbHost(authUser.user.uri);
|
const ldHost = this.utilityService.extractDbHost(authUser.user.uri);
|
||||||
if (!this.utilityService.isFederationAllowedHost(ldHost)) {
|
if (!this.utilityService.isFederationAllowedHost(ldHost)) {
|
||||||
incCounter(this.mIncomingApReject, { reason: 'fed_host_not_allowed' });
|
incCounter(mIncomingApReject, { reason: 'fed_host_not_allowed' });
|
||||||
throw new Bull.UnrecoverableError(`Blocked request: ${ldHost}`);
|
throw new Bull.UnrecoverableError(`Blocked request: ${ldHost}`);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
incCounter(this.mIncomingApReject, { reason: 'ld_signature_unavailable' });
|
incCounter(mIncomingApReject, { reason: 'ld_signature_unavailable' });
|
||||||
throw new Bull.UnrecoverableError(`skip: http-signature verification failed and no LD-Signature. keyId=${signature.keyId}`);
|
throw new Bull.UnrecoverableError(`skip: http-signature verification failed and no LD-Signature. keyId=${signature.keyId}`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -251,7 +252,7 @@ export class InboxProcessorService implements OnApplicationShutdown {
|
||||||
const signerHost = this.utilityService.extractDbHost(authUser.user.uri!);
|
const signerHost = this.utilityService.extractDbHost(authUser.user.uri!);
|
||||||
const activityIdHost = this.utilityService.extractDbHost(activity.id);
|
const activityIdHost = this.utilityService.extractDbHost(activity.id);
|
||||||
if (signerHost !== activityIdHost) {
|
if (signerHost !== activityIdHost) {
|
||||||
incCounter(this.mIncomingApReject, 'host_signature_mismatch');
|
incCounter(mIncomingApReject, 'host_signature_mismatch');
|
||||||
throw new Bull.UnrecoverableError(`skip: signerHost(${signerHost}) !== activity.id host(${activityIdHost}`);
|
throw new Bull.UnrecoverableError(`skip: signerHost(${signerHost}) !== activity.id host(${activityIdHost}`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -279,7 +280,7 @@ export class InboxProcessorService implements OnApplicationShutdown {
|
||||||
this.fetchInstanceMetadataService.fetchInstanceMetadata(i);
|
this.fetchInstanceMetadataService.fetchInstanceMetadata(i);
|
||||||
});
|
});
|
||||||
|
|
||||||
incCounter(this.mIncomingApEvent, {});
|
incCounter(mIncomingApEvent, {});
|
||||||
|
|
||||||
// アクティビティを処理
|
// アクティビティを処理
|
||||||
const begin = +new Date();
|
const begin = +new Date();
|
||||||
|
@ -292,25 +293,25 @@ export class InboxProcessorService implements OnApplicationShutdown {
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
if (e instanceof IdentifiableError) {
|
if (e instanceof IdentifiableError) {
|
||||||
if (e.id === '689ee33f-f97c-479a-ac49-1b9f8140af99') {
|
if (e.id === '689ee33f-f97c-479a-ac49-1b9f8140af99') {
|
||||||
incCounter(this.mIncomingApReject, { reason: 'blocked_notes_with_prohibited_words' });
|
incCounter(mIncomingApReject, { reason: 'blocked_notes_with_prohibited_words' });
|
||||||
return 'blocked notes with prohibited words';
|
return 'blocked notes with prohibited words';
|
||||||
}
|
}
|
||||||
if (e.id === '85ab9bd7-3a41-4530-959d-f07073900109') {
|
if (e.id === '85ab9bd7-3a41-4530-959d-f07073900109') {
|
||||||
incCounter(this.mIncomingApReject, { reason: 'actor_suspended' });
|
incCounter(mIncomingApReject, { reason: 'actor_suspended' });
|
||||||
return 'actor has been suspended';
|
return 'actor has been suspended';
|
||||||
}
|
}
|
||||||
if (e.id === 'd450b8a9-48e4-4dab-ae36-f4db763fda7c') { // invalid Note
|
if (e.id === 'd450b8a9-48e4-4dab-ae36-f4db763fda7c') { // invalid Note
|
||||||
incCounter(this.mIncomingApReject, { reason: 'invalid_note' });
|
incCounter(mIncomingApReject, { reason: 'invalid_note' });
|
||||||
return e.message;
|
return e.message;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
const end = +new Date();
|
const end = +new Date();
|
||||||
observeHistogram(this.mIncomingApProcessingTime, { success: 'false' }, (end - begin) / 1000);
|
observeHistogram(mIncomingApProcessingTime, { success: 'false' }, (end - begin) / 1000);
|
||||||
incCounter(this.mincomingApProcessingError, { reason: 'unknown' });
|
incCounter(mincomingApProcessingError, { reason: 'unknown' });
|
||||||
throw e;
|
throw e;
|
||||||
}
|
}
|
||||||
observeHistogram(this.mIncomingApProcessingTime, { success: 'true' }, (+new Date() - begin) / 1000);
|
observeHistogram(mIncomingApProcessingTime, { success: 'true' }, (+new Date() - begin) / 1000);
|
||||||
incCounter(this.mIncomingApEventAccepted, {});
|
incCounter(mIncomingApEventAccepted, {});
|
||||||
|
|
||||||
return 'ok';
|
return 'ok';
|
||||||
}
|
}
|
||||||
|
|
|
@ -35,7 +35,7 @@ import { makeHstsHook } from './hsts.js';
|
||||||
import { generateCSP } from './csp.js';
|
import { generateCSP } from './csp.js';
|
||||||
import * as prom from 'prom-client';
|
import * as prom from 'prom-client';
|
||||||
import { sanitizeRequestURI } from '@/misc/log-sanitization.js';
|
import { sanitizeRequestURI } from '@/misc/log-sanitization.js';
|
||||||
import { MetricsService } from './api/MetricsService.js';
|
import { metricCounter, metricGauge, metricHistogram, MetricsService } from './api/MetricsService.js';
|
||||||
|
|
||||||
const _dirname = fileURLToPath(new URL('.', import.meta.url));
|
const _dirname = fileURLToPath(new URL('.', import.meta.url));
|
||||||
|
|
||||||
|
@ -55,65 +55,65 @@ function categorizeRequestPath(path: string): 'api' | 'health' | 'vite' | 'other
|
||||||
return 'other';
|
return 'other';
|
||||||
}
|
}
|
||||||
|
|
||||||
@Injectable()
|
const mRequestTime = metricHistogram({
|
||||||
export class ServerService implements OnApplicationShutdown {
|
|
||||||
private logger: Logger;
|
|
||||||
#fastify: FastifyInstance;
|
|
||||||
|
|
||||||
private mRequestTime = new prom.Histogram({
|
|
||||||
name: 'misskey_http_request_duration_seconds',
|
name: 'misskey_http_request_duration_seconds',
|
||||||
help: 'Duration of handling HTTP requests in seconds',
|
help: 'Duration of handling HTTP requests in seconds',
|
||||||
labelNames: ['host', 'cate', 'method', 'path'],
|
labelNames: ['host', 'cate', 'method', 'path'],
|
||||||
buckets: [0.001, 0.1, 0.5, 1, 2, 5],
|
buckets: [0.001, 0.1, 0.5, 1, 2, 5],
|
||||||
});
|
});
|
||||||
|
|
||||||
private mRequestsReceived = new prom.Counter({
|
const mRequestsReceived = metricCounter({
|
||||||
name: 'misskey_http_requests_received_total',
|
name: 'misskey_http_requests_received_total',
|
||||||
help: 'Total number of HTTP requests received',
|
help: 'Total number of HTTP requests received',
|
||||||
labelNames: [],
|
labelNames: [],
|
||||||
});
|
});
|
||||||
|
|
||||||
private mNotFoundServed = new prom.Counter({
|
const mNotFoundServed = metricCounter({
|
||||||
name: 'misskey_http_not_found_served_total',
|
name: 'misskey_http_not_found_served_total',
|
||||||
help: 'Total number of HTTP 404 responses served',
|
help: 'Total number of HTTP 404 responses served',
|
||||||
labelNames: ['method', 'cate'],
|
labelNames: ['method', 'cate'],
|
||||||
});
|
});
|
||||||
|
|
||||||
private mMethodNotAllowedServed = new prom.Counter({
|
const mMethodNotAllowedServed = metricCounter({
|
||||||
name: 'misskey_http_method_not_allowed_served_total',
|
name: 'misskey_http_method_not_allowed_served_total',
|
||||||
help: 'Total number of HTTP 405 responses served',
|
help: 'Total number of HTTP 405 responses served',
|
||||||
labelNames: ['method', 'cate'],
|
labelNames: ['method', 'cate'],
|
||||||
});
|
});
|
||||||
|
|
||||||
private mTooManyRequestsServed = new prom.Counter({
|
const mTooManyRequestsServed = metricCounter({
|
||||||
name: 'misskey_http_too_many_requests_served_total',
|
name: 'misskey_http_too_many_requests_served_total',
|
||||||
help: 'Total number of HTTP 429 responses served',
|
help: 'Total number of HTTP 429 responses served',
|
||||||
labelNames: ['method', 'cate'],
|
labelNames: ['method', 'cate'],
|
||||||
});
|
});
|
||||||
|
|
||||||
private mAggregateRequestsServed = new prom.Counter({
|
const mAggregateRequestsServed = metricCounter({
|
||||||
name: 'misskey_http_requests_served_total',
|
name: 'misskey_http_requests_served_total',
|
||||||
help: 'Total number of HTTP requests served including invalid requests',
|
help: 'Total number of HTTP requests served including invalid requests',
|
||||||
labelNames: ['host', 'cate', 'status'],
|
labelNames: ['host', 'cate', 'status'],
|
||||||
});
|
});
|
||||||
|
|
||||||
private mRequestsServedByPath = new prom.Counter({
|
const mRequestsServedByPath = metricCounter({
|
||||||
name: 'misskey_http_requests_served_by_path',
|
name: 'misskey_http_requests_served_by_path',
|
||||||
help: 'Total number of HTTP requests served',
|
help: 'Total number of HTTP requests served',
|
||||||
labelNames: ['host', 'cate', 'method', 'path', 'status'],
|
labelNames: ['host', 'cate', 'method', 'path', 'status'],
|
||||||
});
|
});
|
||||||
|
|
||||||
private mFatalErrorCount = new prom.Counter({
|
const mFatalErrorCount = metricCounter({
|
||||||
name: 'misskey_fatal_http_errors_total',
|
name: 'misskey_fatal_http_errors_total',
|
||||||
help: 'Total number of HTTP errors that propagate to the top level',
|
help: 'Total number of HTTP errors that propagate to the top level',
|
||||||
labelNames: ['host', 'cate', 'method', 'path'],
|
labelNames: ['host', 'cate', 'method', 'path'],
|
||||||
});
|
});
|
||||||
|
|
||||||
private mLastSuccessfulRequest = new prom.Gauge({
|
const mLastSuccessfulRequest = metricGauge({
|
||||||
name: 'misskey_http_last_successful_request_timestamp_seconds',
|
name: 'misskey_http_last_successful_request_timestamp_seconds',
|
||||||
help: 'Unix Timestamp of the last successful HTTP request',
|
help: 'Unix Timestamp of the last successful HTTP request',
|
||||||
labelNames: [],
|
labelNames: [],
|
||||||
});
|
});
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class ServerService implements OnApplicationShutdown {
|
||||||
|
private logger: Logger;
|
||||||
|
#fastify: FastifyInstance;
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
@Inject(DI.config)
|
@Inject(DI.config)
|
||||||
|
@ -160,14 +160,14 @@ export class ServerService implements OnApplicationShutdown {
|
||||||
if (this.config.prometheusMetrics?.enable) {
|
if (this.config.prometheusMetrics?.enable) {
|
||||||
fastify.addHook('onRequest', (_request, reply, done) => {
|
fastify.addHook('onRequest', (_request, reply, done) => {
|
||||||
reply.header('x-request-received', (+new Date()).toString());
|
reply.header('x-request-received', (+new Date()).toString());
|
||||||
this.mRequestsReceived.inc();
|
mRequestsReceived?.inc();
|
||||||
done();
|
done();
|
||||||
});
|
});
|
||||||
|
|
||||||
fastify.addHook('onError', (request, _reply, error, done) => {
|
fastify.addHook('onError', (request, _reply, error, done) => {
|
||||||
const url = new URL(request.url, this.config.url);
|
const url = new URL(request.url, this.config.url);
|
||||||
const logPath = sanitizeRequestURI(url.pathname);
|
const logPath = sanitizeRequestURI(url.pathname);
|
||||||
this.mFatalErrorCount.inc({
|
mFatalErrorCount?.inc({
|
||||||
host: request.hostname,
|
host: request.hostname,
|
||||||
method: request.method,
|
method: request.method,
|
||||||
path: logPath,
|
path: logPath,
|
||||||
|
@ -182,14 +182,14 @@ export class ServerService implements OnApplicationShutdown {
|
||||||
const cate = categorizeRequestPath(logPath);
|
const cate = categorizeRequestPath(logPath);
|
||||||
const received = reply.getHeader('x-request-received') as string;
|
const received = reply.getHeader('x-request-received') as string;
|
||||||
|
|
||||||
this.mAggregateRequestsServed.inc({
|
mAggregateRequestsServed?.inc({
|
||||||
host: request.hostname,
|
host: request.hostname,
|
||||||
cate,
|
cate,
|
||||||
status: reply.statusCode,
|
status: reply.statusCode,
|
||||||
});
|
});
|
||||||
|
|
||||||
if (reply.statusCode === 429) {
|
if (reply.statusCode === 429) {
|
||||||
this.mTooManyRequestsServed.inc({
|
mTooManyRequestsServed?.inc({
|
||||||
method: request.method,
|
method: request.method,
|
||||||
cate,
|
cate,
|
||||||
});
|
});
|
||||||
|
@ -199,14 +199,14 @@ export class ServerService implements OnApplicationShutdown {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (reply.statusCode === 404) {
|
if (reply.statusCode === 404) {
|
||||||
this.mNotFoundServed.inc({
|
mNotFoundServed?.inc({
|
||||||
method: request.method,
|
method: request.method,
|
||||||
cate,
|
cate,
|
||||||
});
|
});
|
||||||
|
|
||||||
if (received) {
|
if (received) {
|
||||||
const duration = (+new Date()) - parseInt(received);
|
const duration = (+new Date()) - parseInt(received);
|
||||||
this.mRequestTime.observe({
|
mRequestTime?.observe({
|
||||||
host: request.hostname,
|
host: request.hostname,
|
||||||
method: request.method,
|
method: request.method,
|
||||||
cate,
|
cate,
|
||||||
|
@ -218,7 +218,7 @@ export class ServerService implements OnApplicationShutdown {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (reply.statusCode === 405) {
|
if (reply.statusCode === 405) {
|
||||||
this.mMethodNotAllowedServed.inc({
|
mMethodNotAllowedServed?.inc({
|
||||||
method: request.method,
|
method: request.method,
|
||||||
cate,
|
cate,
|
||||||
});
|
});
|
||||||
|
@ -229,7 +229,7 @@ export class ServerService implements OnApplicationShutdown {
|
||||||
if (received) {
|
if (received) {
|
||||||
const duration = (+new Date()) - parseInt(received);
|
const duration = (+new Date()) - parseInt(received);
|
||||||
|
|
||||||
this.mRequestTime.observe({
|
mRequestTime?.observe({
|
||||||
host: request.hostname,
|
host: request.hostname,
|
||||||
method: request.method,
|
method: request.method,
|
||||||
cate,
|
cate,
|
||||||
|
@ -243,10 +243,10 @@ export class ServerService implements OnApplicationShutdown {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (reply.statusCode <= 299) {
|
if (reply.statusCode <= 299) {
|
||||||
this.mLastSuccessfulRequest.set(+new Date() / 1000);
|
mLastSuccessfulRequest?.set(+new Date() / 1000);
|
||||||
}
|
}
|
||||||
|
|
||||||
this.mRequestsServedByPath.inc({
|
mRequestsServedByPath?.inc({
|
||||||
host: request.hostname,
|
host: request.hostname,
|
||||||
method: request.method,
|
method: request.method,
|
||||||
path: logPath,
|
path: logPath,
|
||||||
|
|
|
@ -14,6 +14,7 @@ import type { MiApp } from '@/models/App.js';
|
||||||
import { CacheService } from '@/core/CacheService.js';
|
import { CacheService } from '@/core/CacheService.js';
|
||||||
import isNativeToken from '@/misc/is-native-token.js';
|
import isNativeToken from '@/misc/is-native-token.js';
|
||||||
import { bindThis } from '@/decorators.js';
|
import { bindThis } from '@/decorators.js';
|
||||||
|
import { metricCounter } from './MetricsService.js';
|
||||||
|
|
||||||
export class AuthenticationError extends Error {
|
export class AuthenticationError extends Error {
|
||||||
constructor(message: string) {
|
constructor(message: string) {
|
||||||
|
@ -22,15 +23,15 @@ export class AuthenticationError extends Error {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Injectable()
|
const mAuthenticationFailureCounter = metricCounter({
|
||||||
export class AuthenticateService implements OnApplicationShutdown {
|
|
||||||
private appCache: MemoryKVCache<MiApp>;
|
|
||||||
|
|
||||||
private mAuthenticationFailureCounter = new prom.Counter({
|
|
||||||
name: 'misskey_authentication_failure_total',
|
name: 'misskey_authentication_failure_total',
|
||||||
help: 'Total number of authentication failures',
|
help: 'Total number of authentication failures',
|
||||||
labelNames: ['cred_ty'],
|
labelNames: ['cred_ty'],
|
||||||
});
|
});
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class AuthenticateService implements OnApplicationShutdown {
|
||||||
|
private appCache: MemoryKVCache<MiApp>;
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
@Inject(DI.usersRepository)
|
@Inject(DI.usersRepository)
|
||||||
|
@ -58,7 +59,7 @@ export class AuthenticateService implements OnApplicationShutdown {
|
||||||
() => this.usersRepository.findOneBy({ token }) as Promise<MiLocalUser | null>);
|
() => this.usersRepository.findOneBy({ token }) as Promise<MiLocalUser | null>);
|
||||||
|
|
||||||
if (user == null) {
|
if (user == null) {
|
||||||
this.mAuthenticationFailureCounter.inc({ cred_ty: 'native' });
|
mAuthenticationFailureCounter?.inc({ cred_ty: 'native' });
|
||||||
throw new AuthenticationError('user not found');
|
throw new AuthenticationError('user not found');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -73,7 +74,7 @@ export class AuthenticateService implements OnApplicationShutdown {
|
||||||
});
|
});
|
||||||
|
|
||||||
if (accessToken == null) {
|
if (accessToken == null) {
|
||||||
this.mAuthenticationFailureCounter.inc({ cred_ty: 'access_token' });
|
mAuthenticationFailureCounter?.inc({ cred_ty: 'access_token' });
|
||||||
throw new AuthenticationError('invalid signature');
|
throw new AuthenticationError('invalid signature');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -5,6 +5,30 @@ import type { Config } from '@/config.js';
|
||||||
import { bindThis } from "@/decorators.js";
|
import { bindThis } from "@/decorators.js";
|
||||||
import type { FastifyInstance, FastifyPluginOptions } from "fastify";
|
import type { FastifyInstance, FastifyPluginOptions } from "fastify";
|
||||||
|
|
||||||
|
export function metricGauge<K extends string>(conf: prom.GaugeConfiguration<K>) : prom.Gauge<K> | null {
|
||||||
|
if (!process.env.RUN_MODE) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return new prom.Gauge(conf);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function metricCounter<K extends string>(conf: prom.CounterConfiguration<K>) : prom.Counter<K> | null {
|
||||||
|
if (!process.env.RUN_MODE) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return new prom.Counter(conf);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function metricHistogram<K extends string>(conf: prom.HistogramConfiguration<K>) : prom.Histogram<K> | null {
|
||||||
|
if (!process.env.RUN_MODE) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return new prom.Histogram(conf);
|
||||||
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* SPDX-FileCopyrightText: syuilo and misskey-project and yumechi
|
* SPDX-FileCopyrightText: syuilo and misskey-project and yumechi
|
||||||
* SPDX-License-Identifier: AGPL-3.0-only
|
* SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
|
|
@ -29,15 +29,16 @@ import { RateLimiterService } from './RateLimiterService.js';
|
||||||
import { SigninService } from './SigninService.js';
|
import { SigninService } from './SigninService.js';
|
||||||
import type { AuthenticationResponseJSON } from '@simplewebauthn/types';
|
import type { AuthenticationResponseJSON } from '@simplewebauthn/types';
|
||||||
import type { FastifyReply, FastifyRequest } from 'fastify';
|
import type { FastifyReply, FastifyRequest } from 'fastify';
|
||||||
|
import { metricCounter } from './MetricsService.js';
|
||||||
|
|
||||||
@Injectable()
|
const mSigninFailureCounter = metricCounter({
|
||||||
export class SigninApiService {
|
|
||||||
private mSigninFailureCounter = new prom.Counter({
|
|
||||||
name: 'misskey_misskey_signin_failure',
|
name: 'misskey_misskey_signin_failure',
|
||||||
help: 'The number of failed sign-ins',
|
help: 'The number of failed sign-ins',
|
||||||
labelNames: ['reason'],
|
labelNames: ['reason'],
|
||||||
});
|
});
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class SigninApiService {
|
||||||
constructor(
|
constructor(
|
||||||
@Inject(DI.config)
|
@Inject(DI.config)
|
||||||
private config: Config,
|
private config: Config,
|
||||||
|
@ -100,7 +101,7 @@ export class SigninApiService {
|
||||||
// not more than 1 attempt per second and not more than 10 attempts per hour
|
// not more than 1 attempt per second and not more than 10 attempts per hour
|
||||||
await this.rateLimiterService.limit({ key: 'signin', duration: 60 * 60 * 1000, max: 10, minInterval: 1000 }, getIpHash(request.ip));
|
await this.rateLimiterService.limit({ key: 'signin', duration: 60 * 60 * 1000, max: 10, minInterval: 1000 }, getIpHash(request.ip));
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
this.mSigninFailureCounter.inc({ reason: 'rate_limit' });
|
mSigninFailureCounter?.inc({ reason: 'rate_limit' });
|
||||||
reply.code(429);
|
reply.code(429);
|
||||||
return {
|
return {
|
||||||
error: {
|
error: {
|
||||||
|
@ -112,13 +113,13 @@ export class SigninApiService {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (typeof username !== 'string') {
|
if (typeof username !== 'string') {
|
||||||
this.mSigninFailureCounter.inc({ reason: 'bad_form' });
|
mSigninFailureCounter?.inc({ reason: 'bad_form' });
|
||||||
reply.code(400);
|
reply.code(400);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (token != null && typeof token !== 'string') {
|
if (token != null && typeof token !== 'string') {
|
||||||
this.mSigninFailureCounter.inc({ reason: 'bad_form' });
|
mSigninFailureCounter?.inc({ reason: 'bad_form' });
|
||||||
reply.code(400);
|
reply.code(400);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -130,14 +131,14 @@ export class SigninApiService {
|
||||||
}) as MiLocalUser;
|
}) as MiLocalUser;
|
||||||
|
|
||||||
if (user == null) {
|
if (user == null) {
|
||||||
this.mSigninFailureCounter.inc({ reason: 'user_not_found' });
|
mSigninFailureCounter?.inc({ reason: 'user_not_found' });
|
||||||
return error(404, {
|
return error(404, {
|
||||||
id: '6cc579cc-885d-43d8-95c2-b8c7fc963280',
|
id: '6cc579cc-885d-43d8-95c2-b8c7fc963280',
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
if (user.isSuspended) {
|
if (user.isSuspended) {
|
||||||
this.mSigninFailureCounter.inc({ reason: 'user_suspended' });
|
mSigninFailureCounter?.inc({ reason: 'user_suspended' });
|
||||||
return error(403, {
|
return error(403, {
|
||||||
id: 'e03a5f46-d309-4865-9b69-56282d94e1eb',
|
id: 'e03a5f46-d309-4865-9b69-56282d94e1eb',
|
||||||
});
|
});
|
||||||
|
@ -162,7 +163,7 @@ export class SigninApiService {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (typeof password !== 'string') {
|
if (typeof password !== 'string') {
|
||||||
this.mSigninFailureCounter.inc({ reason: 'bad_form' });
|
mSigninFailureCounter?.inc({ reason: 'bad_form' });
|
||||||
reply.code(400);
|
reply.code(400);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -180,7 +181,7 @@ export class SigninApiService {
|
||||||
success: false,
|
success: false,
|
||||||
});
|
});
|
||||||
|
|
||||||
this.mSigninFailureCounter.inc({ reason: failure?.id ?? `unknown_error_${status ?? 500}` });
|
mSigninFailureCounter?.inc({ reason: failure?.id ?? `unknown_error_${status ?? 500}` });
|
||||||
return error(status ?? 500, failure ?? { id: '4e30e80c-e338-45a0-8c8f-44455efa3b76' });
|
return error(status ?? 500, failure ?? { id: '4e30e80c-e338-45a0-8c8f-44455efa3b76' });
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -188,35 +189,35 @@ export class SigninApiService {
|
||||||
if (process.env.NODE_ENV !== 'test') {
|
if (process.env.NODE_ENV !== 'test') {
|
||||||
if (this.meta.enableHcaptcha && this.meta.hcaptchaSecretKey) {
|
if (this.meta.enableHcaptcha && this.meta.hcaptchaSecretKey) {
|
||||||
await this.captchaService.verifyHcaptcha(this.meta.hcaptchaSecretKey, body['hcaptcha-response']).catch(err => {
|
await this.captchaService.verifyHcaptcha(this.meta.hcaptchaSecretKey, body['hcaptcha-response']).catch(err => {
|
||||||
this.mSigninFailureCounter.inc({ reason: 'captcha_verification_failed_hcaptcha' });
|
mSigninFailureCounter?.inc({ reason: 'captcha_verification_failed_hcaptcha' });
|
||||||
throw new FastifyReplyError(400, err);
|
throw new FastifyReplyError(400, err);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
if (this.meta.enableMcaptcha && this.meta.mcaptchaSecretKey && this.meta.mcaptchaSitekey && this.meta.mcaptchaInstanceUrl) {
|
if (this.meta.enableMcaptcha && this.meta.mcaptchaSecretKey && this.meta.mcaptchaSitekey && this.meta.mcaptchaInstanceUrl) {
|
||||||
await this.captchaService.verifyMcaptcha(this.meta.mcaptchaSecretKey, this.meta.mcaptchaSitekey, this.meta.mcaptchaInstanceUrl, body['m-captcha-response']).catch(err => {
|
await this.captchaService.verifyMcaptcha(this.meta.mcaptchaSecretKey, this.meta.mcaptchaSitekey, this.meta.mcaptchaInstanceUrl, body['m-captcha-response']).catch(err => {
|
||||||
this.mSigninFailureCounter.inc({ reason: 'captcha_verification_failed_mcaptcha' });
|
mSigninFailureCounter?.inc({ reason: 'captcha_verification_failed_mcaptcha' });
|
||||||
throw new FastifyReplyError(400, err);
|
throw new FastifyReplyError(400, err);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
if (this.meta.enableRecaptcha && this.meta.recaptchaSecretKey) {
|
if (this.meta.enableRecaptcha && this.meta.recaptchaSecretKey) {
|
||||||
await this.captchaService.verifyRecaptcha(this.meta.recaptchaSecretKey, body['g-recaptcha-response']).catch(err => {
|
await this.captchaService.verifyRecaptcha(this.meta.recaptchaSecretKey, body['g-recaptcha-response']).catch(err => {
|
||||||
this.mSigninFailureCounter.inc({ reason: 'captcha_verification_failed_recaptcha' });
|
mSigninFailureCounter?.inc({ reason: 'captcha_verification_failed_recaptcha' });
|
||||||
throw new FastifyReplyError(400, err);
|
throw new FastifyReplyError(400, err);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
if (this.meta.enableTurnstile && this.meta.turnstileSecretKey) {
|
if (this.meta.enableTurnstile && this.meta.turnstileSecretKey) {
|
||||||
await this.captchaService.verifyTurnstile(this.meta.turnstileSecretKey, body['turnstile-response']).catch(err => {
|
await this.captchaService.verifyTurnstile(this.meta.turnstileSecretKey, body['turnstile-response']).catch(err => {
|
||||||
this.mSigninFailureCounter.inc({ reason: 'captcha_verification_failed_turnstile' });
|
mSigninFailureCounter?.inc({ reason: 'captcha_verification_failed_turnstile' });
|
||||||
throw new FastifyReplyError(400, err);
|
throw new FastifyReplyError(400, err);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
if (this.meta.enableTestcaptcha) {
|
if (this.meta.enableTestcaptcha) {
|
||||||
await this.captchaService.verifyTestcaptcha(body['testcaptcha-response']).catch(err => {
|
await this.captchaService.verifyTestcaptcha(body['testcaptcha-response']).catch(err => {
|
||||||
this.mSigninFailureCounter.inc({ reason: 'captcha_verification_failed_testcaptcha' });
|
mSigninFailureCounter?.inc({ reason: 'captcha_verification_failed_testcaptcha' });
|
||||||
throw new FastifyReplyError(400, err);
|
throw new FastifyReplyError(400, err);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
|
@ -5,7 +5,6 @@ import { NestFactory } from '@nestjs/core';
|
||||||
import { MainModule } from '@/MainModule.js';
|
import { MainModule } from '@/MainModule.js';
|
||||||
import { ServerService } from '@/server/ServerService.js';
|
import { ServerService } from '@/server/ServerService.js';
|
||||||
import { loadConfig } from '@/config.js';
|
import { loadConfig } from '@/config.js';
|
||||||
import { NestLogger } from '@/NestLogger.js';
|
|
||||||
import { INestApplicationContext } from '@nestjs/common';
|
import { INestApplicationContext } from '@nestjs/common';
|
||||||
|
|
||||||
const config = loadConfig();
|
const config = loadConfig();
|
||||||
|
@ -22,10 +21,8 @@ let serverService: ServerService;
|
||||||
async function launch() {
|
async function launch() {
|
||||||
await killTestServer();
|
await killTestServer();
|
||||||
|
|
||||||
console.log('starting application...');
|
|
||||||
|
|
||||||
app = await NestFactory.createApplicationContext(MainModule, {
|
app = await NestFactory.createApplicationContext(MainModule, {
|
||||||
logger: new NestLogger(),
|
logger: ["debug", "log", "error", "warn", "verbose"],
|
||||||
});
|
});
|
||||||
serverService = app.get(ServerService);
|
serverService = app.get(ServerService);
|
||||||
await serverService.launch();
|
await serverService.launch();
|
||||||
|
@ -84,7 +81,7 @@ async function startControllerEndpoints(port = config.port + 1000) {
|
||||||
console.log('starting application...');
|
console.log('starting application...');
|
||||||
|
|
||||||
app = await NestFactory.createApplicationContext(MainModule, {
|
app = await NestFactory.createApplicationContext(MainModule, {
|
||||||
logger: new NestLogger(),
|
logger: ["debug", "log", "error", "warn", "verbose"],
|
||||||
});
|
});
|
||||||
serverService = app.get(ServerService);
|
serverService = app.get(ServerService);
|
||||||
await serverService.launch();
|
await serverService.launch();
|
||||||
|
|
|
@ -12,6 +12,9 @@ importers:
|
||||||
|
|
||||||
.:
|
.:
|
||||||
dependencies:
|
dependencies:
|
||||||
|
cross-env:
|
||||||
|
specifier: 7.0.3
|
||||||
|
version: 7.0.3
|
||||||
cssnano:
|
cssnano:
|
||||||
specifier: 6.1.2
|
specifier: 6.1.2
|
||||||
version: 6.1.2(postcss@8.4.49)
|
version: 6.1.2(postcss@8.4.49)
|
||||||
|
@ -62,9 +65,6 @@ importers:
|
||||||
'@typescript-eslint/parser':
|
'@typescript-eslint/parser':
|
||||||
specifier: 7.17.0
|
specifier: 7.17.0
|
||||||
version: 7.17.0(eslint@9.14.0)(typescript@5.6.3)
|
version: 7.17.0(eslint@9.14.0)(typescript@5.6.3)
|
||||||
cross-env:
|
|
||||||
specifier: 7.0.3
|
|
||||||
version: 7.0.3
|
|
||||||
cypress:
|
cypress:
|
||||||
specifier: 13.15.2
|
specifier: 13.15.2
|
||||||
version: 13.15.2
|
version: 13.15.2
|
||||||
|
|
Loading…
Reference in a new issue