From 13e50cd8d9e1f8e2d91c0ca387206b904c495fe5 Mon Sep 17 00:00:00 2001 From: eternal-flame-AD Date: Sat, 16 Nov 2024 17:10:30 -0600 Subject: [PATCH 1/3] =?UTF-8?q?fix(backend):=20=E3=82=A2=E3=83=8A=E3=82=A6?= =?UTF-8?q?=E3=83=B3=E3=82=B9=E3=83=A1=E3=83=B3=E3=83=88=E3=82=92=E4=BD=9C?= =?UTF-8?q?=E6=88=90=E3=81=A8=E3=81=8D=E3=81=AB=E7=94=BB=E5=83=8FURL?= =?UTF-8?q?=E3=82=92=E5=BE=8C=E6=82=94=E3=81=A7=E3=81=8D=E3=81=AA=E3=81=84?= =?UTF-8?q?=E3=81=AE=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: eternal-flame-AD --- CHANGELOG.md | 4 ++++ packages/backend/src/core/AnnouncementService.ts | 2 +- .../src/server/api/endpoints/admin/announcements/create.ts | 5 +++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index af8e6acde9..dbf0744cbe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ PgroongaのCWサーチ (github.com/paricafe/misskey#d30db97b59d264450901c1dd86808dcb43875ea9) +### 2024.11.0-yumechinokuni.4p2 + +- fix(backend): アナウンスメントを作成ときにWebUIフォームの画像URLを後悔できないのを修正 (/admin/announcement/create) + ## 2024.11.0-yumechinokuni.3 - Security: CSPの設定を強化 diff --git a/packages/backend/src/core/AnnouncementService.ts b/packages/backend/src/core/AnnouncementService.ts index d4fcf19439..a9f6731977 100644 --- a/packages/backend/src/core/AnnouncementService.ts +++ b/packages/backend/src/core/AnnouncementService.ts @@ -72,7 +72,7 @@ export class AnnouncementService { updatedAt: null, title: values.title, text: values.text, - imageUrl: values.imageUrl, + imageUrl: values.imageUrl || null, icon: values.icon, display: values.display, forExistingUsers: values.forExistingUsers, diff --git a/packages/backend/src/server/api/endpoints/admin/announcements/create.ts b/packages/backend/src/server/api/endpoints/admin/announcements/create.ts index 2dae1df87d..b8bfda73a4 100644 --- a/packages/backend/src/server/api/endpoints/admin/announcements/create.ts +++ b/packages/backend/src/server/api/endpoints/admin/announcements/create.ts @@ -55,7 +55,7 @@ export const paramDef = { properties: { title: { type: 'string', minLength: 1 }, text: { type: 'string', minLength: 1 }, - imageUrl: { type: 'string', nullable: true, minLength: 1 }, + imageUrl: { type: 'string', nullable: true, minLength: 0 }, icon: { type: 'string', enum: ['info', 'warning', 'error', 'success'], default: 'info' }, display: { type: 'string', enum: ['normal', 'banner', 'dialog'], default: 'normal' }, forExistingUsers: { type: 'boolean', default: false }, @@ -76,7 +76,8 @@ export default class extends Endpoint { // eslint- updatedAt: null, title: ps.title, text: ps.text, - imageUrl: ps.imageUrl, + /* eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- 空の文字列の場合、nullを渡すようにするため */ + imageUrl: ps.imageUrl || null, icon: ps.icon, display: ps.display, forExistingUsers: ps.forExistingUsers, From 4d44adfaa9fffb0141d23a6ba01a4a429d6aab21 Mon Sep 17 00:00:00 2001 From: eternal-flame-AD Date: Sat, 16 Nov 2024 23:00:06 -0600 Subject: [PATCH 2/3] prometheus - stage 1 deployment Signed-off-by: eternal-flame-AD --- .config/cypress-devcontainer.yml | 7 + .config/docker_example.yml | 7 + .config/example.yml | 7 + packages/backend/package.json | 3 +- packages/backend/scripts/check_connect.js | 9 +- packages/backend/src/GlobalModule.ts | 2 +- packages/backend/src/boot/master.ts | 21 ++ packages/backend/src/boot/worker.ts | 7 + packages/backend/src/config.ts | 8 + packages/backend/src/core/QueueModule.ts | 33 +++- .../src/core/activitypub/ApInboxService.ts | 22 +++ packages/backend/src/misc/log-sanitization.ts | 42 ++++ packages/backend/src/postgres.ts | 103 +++++++++- .../src/queue/QueueProcessorService.ts | 41 +++- packages/backend/src/queue/metrics.ts | 31 +++ .../queue/processors/InboxProcessorService.ts | 78 +++++++- packages/backend/src/server/ServerModule.ts | 2 + packages/backend/src/server/ServerService.ts | 182 ++++++++++++++++++ .../src/server/api/AuthenticateService.ts | 9 + .../backend/src/server/api/MetricsService.ts | 52 +++++ .../src/server/api/SigninApiService.ts | 19 ++ packages/frontend/.storybook/fakes.ts | 2 +- packages/misskey-js/etc/misskey-js.api.md | 16 -- packages/misskey-js/src/autogen/models.ts | 4 - packages/misskey-js/src/autogen/types.ts | 26 --- pnpm-lock.yaml | 24 +++ yume-mods/etc/config.alloy | 18 ++ 27 files changed, 703 insertions(+), 72 deletions(-) create mode 100644 packages/backend/src/misc/log-sanitization.ts create mode 100644 packages/backend/src/queue/metrics.ts create mode 100644 packages/backend/src/server/api/MetricsService.ts create mode 100644 yume-mods/etc/config.alloy diff --git a/.config/cypress-devcontainer.yml b/.config/cypress-devcontainer.yml index 179ce77dc9..8c908769aa 100644 --- a/.config/cypress-devcontainer.yml +++ b/.config/cypress-devcontainer.yml @@ -153,6 +153,13 @@ redis: id: 'aidx' +# ┌──────────┐ +#───┘ Metrics └────────────────────────────────────────── + +#prometheusMetrics: +# enable: false +# scrapeToken: '' # Set non-empty to require a bearer token for scraping + # ┌────────────────┐ #───┘ Error tracking └────────────────────────────────────────── diff --git a/.config/docker_example.yml b/.config/docker_example.yml index 35dfa5ded3..07f12516e9 100644 --- a/.config/docker_example.yml +++ b/.config/docker_example.yml @@ -147,6 +147,13 @@ redis: id: 'aidx' +# ┌──────────┐ +#───┘ Metrics └────────────────────────────────────────── + +#prometheusMetrics: +# enable: false +# scrapeToken: '' # Set non-empty to require a bearer token for scraping + # ┌────────────────┐ #───┘ Error tracking └────────────────────────────────────────── diff --git a/.config/example.yml b/.config/example.yml index ef0e30d4e1..4a8d7ebeec 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -229,6 +229,13 @@ redis: id: 'aidx' +# ┌──────────┐ +#───┘ Metrics └────────────────────────────────────────── + +#prometheusMetrics: +# enable: false +# scrapeToken: '' # Set non-empty to require a bearer token for scraping + # ┌────────────────┐ #───┘ Error tracking └────────────────────────────────────────── diff --git a/packages/backend/package.json b/packages/backend/package.json index 5055f50235..e6213c4722 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -134,8 +134,8 @@ "json5": "2.2.3", "jsonld": "8.3.2", "jsrsasign": "11.1.0", - "meilisearch": "0.45.0", "juice": "11.0.0", + "meilisearch": "0.45.0", "mfm-js": "0.24.0", "microformats-parser": "2.0.2", "mime-types": "2.1.35", @@ -156,6 +156,7 @@ "pg": "8.13.1", "pkce-challenge": "4.1.0", "probe-image-size": "7.2.3", + "prom-client": "^15.1.3", "promise-limit": "2.7.0", "pug": "3.0.3", "punycode": "2.3.1", diff --git a/packages/backend/scripts/check_connect.js b/packages/backend/scripts/check_connect.js index bb149444b5..5d6d2fca49 100644 --- a/packages/backend/scripts/check_connect.js +++ b/packages/backend/scripts/check_connect.js @@ -7,6 +7,11 @@ import Redis from 'ioredis'; import { loadConfig } from '../built/config.js'; import { createPostgresDataSource } from '../built/postgres.js'; +const timeout = setTimeout(() => { + console.error('Timeout while connecting to databases.'); + process.exit(1); +}, 120000); + const config = loadConfig(); async function connectToPostgres() { @@ -50,7 +55,9 @@ const promises = Array ])) .map(connectToRedis) .concat([ - connectToPostgres() + connectToPostgres().then(() => { console.log('Connected to PostgreSQL.'); }), ]); await Promise.allSettled(promises); + +clearTimeout(timeout); diff --git a/packages/backend/src/GlobalModule.ts b/packages/backend/src/GlobalModule.ts index 6ae8ccfbb3..038c857fe4 100644 --- a/packages/backend/src/GlobalModule.ts +++ b/packages/backend/src/GlobalModule.ts @@ -24,7 +24,7 @@ const $config: Provider = { const $db: Provider = { provide: DI.db, useFactory: async (config) => { - const db = createPostgresDataSource(config); + const db = createPostgresDataSource(config, true); return await db.initialize(); }, inject: [DI.config], diff --git a/packages/backend/src/boot/master.ts b/packages/backend/src/boot/master.ts index 4bc5c799cf..551ff984b1 100644 --- a/packages/backend/src/boot/master.ts +++ b/packages/backend/src/boot/master.ts @@ -7,9 +7,11 @@ import * as fs from 'node:fs'; import { fileURLToPath } from 'node:url'; import { dirname } from 'node:path'; import * as os from 'node:os'; +import * as prom from 'prom-client'; import cluster from 'node:cluster'; import chalk from 'chalk'; import chalkTemplate from 'chalk-template'; +import { collectDefaultMetrics } from 'prom-client'; import * as Sentry from '@sentry/node'; import { nodeProfilingIntegration } from '@sentry/profiling-node'; import Logger from '@/logger.js'; @@ -29,6 +31,18 @@ const bootLogger = logger.createSubLogger('boot', 'magenta'); const themeColor = chalk.hex('#86b300'); +const mBuildInfo = new prom.Gauge({ + name: 'misskey_build_info', + help: 'Misskey build information', + labelNames: ['gitCommit', 'gitDescribe', 'node_version'] +}); + +mBuildInfo.set({ + gitCommit: meta.gitCommit || 'unknown', + gitDescribe: meta.gitDescribe || 'unknown', + node_version: process.version +}, 1); + function greet() { if (!envOption.quiet) { //#region Misskey logo @@ -64,6 +78,13 @@ export async function masterMain() { await showMachineInfo(bootLogger); showNodejsVersion(); config = loadConfigBoot(); + + collectDefaultMetrics({ + labels: { + cluster_type: 'master', + } + }); + //await connectDb(); if (config.pidFile) fs.writeFileSync(config.pidFile, process.pid.toString()); } catch (e) { diff --git a/packages/backend/src/boot/worker.ts b/packages/backend/src/boot/worker.ts index 5d4a15b29f..a4791cf062 100644 --- a/packages/backend/src/boot/worker.ts +++ b/packages/backend/src/boot/worker.ts @@ -4,6 +4,7 @@ */ import cluster from 'node:cluster'; +import { collectDefaultMetrics } from 'prom-client'; import * as Sentry from '@sentry/node'; import { nodeProfilingIntegration } from '@sentry/profiling-node'; import { envOption } from '@/env.js'; @@ -16,6 +17,12 @@ import { jobQueue, server } from './common.js'; export async function workerMain() { const config = loadConfig(); + collectDefaultMetrics({ + labels: { + cluster_type: 'worker', + } + }); + if (config.sentryForBackend) { Sentry.init({ integrations: [ diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index 035c9c6331..b678eef293 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -72,6 +72,9 @@ type Source = { index: string; scope?: 'local' | 'global' | string[]; }; + + prometheusMetrics?: { enable: boolean, scrapeToken?: string }; + sentryForBackend?: { options: Partial; enableNodeProfiling: boolean; }; sentryForFrontend?: { options: Partial }; @@ -199,8 +202,12 @@ export type Config = { redisForJobQueue: RedisOptions & RedisOptionsSource; redisForTimelines: RedisOptions & RedisOptionsSource; redisForReactions: RedisOptions & RedisOptionsSource; + + prometheusMetrics : { enable: boolean, scrapeToken?: string } | undefined; + sentryForBackend: { options: Partial; enableNodeProfiling: boolean; } | undefined; sentryForFrontend: { options: Partial } | undefined; + perChannelMaxNoteCacheCount: number; perUserNotificationsMaxCount: number; deactivateAntennaThreshold: number; @@ -295,6 +302,7 @@ export function loadConfig(): Config { redisForJobQueue: config.redisForJobQueue ? convertRedisOptions(config.redisForJobQueue, host) : redis, redisForTimelines: config.redisForTimelines ? convertRedisOptions(config.redisForTimelines, host) : redis, redisForReactions: config.redisForReactions ? convertRedisOptions(config.redisForReactions, host) : redis, + prometheusMetrics: config.prometheusMetrics, sentryForBackend: config.sentryForBackend, sentryForFrontend: config.sentryForFrontend, id: config.id, diff --git a/packages/backend/src/core/QueueModule.ts b/packages/backend/src/core/QueueModule.ts index b10b8e5899..3fb05cda3b 100644 --- a/packages/backend/src/core/QueueModule.ts +++ b/packages/backend/src/core/QueueModule.ts @@ -18,6 +18,7 @@ import { SystemWebhookDeliverJobData, } from '../queue/types.js'; import type { Provider } from '@nestjs/common'; +import { mActiveJobs, mDelayedJobs, mJobReceivedCounter, mWaitingJobs } from '@/queue/metrics.js'; export type SystemQueue = Bull.Queue>; export type EndedPollNotificationQueue = Bull.Queue; @@ -29,57 +30,71 @@ export type ObjectStorageQueue = Bull.Queue; export type UserWebhookDeliverQueue = Bull.Queue; export type SystemWebhookDeliverQueue = Bull.Queue; +function withMetrics(queue: Bull.Queue): Bull.Queue { + setInterval(async () => { + mActiveJobs.set({ queue: queue.name }, await queue.getActiveCount()); + mDelayedJobs.set({ queue: queue.name }, await queue.getDelayedCount()); + mWaitingJobs.set({ queue: queue.name }, await queue.getWaitingCount()); + }, 2000); + + queue.on('waiting', () => { + mJobReceivedCounter.inc({ queue: queue.name }); + }); + + return queue; +} + const $system: Provider = { provide: 'queue:system', - useFactory: (config: Config) => new Bull.Queue(QUEUE.SYSTEM, baseQueueOptions(config, QUEUE.SYSTEM)), + useFactory: (config: Config) => withMetrics(new Bull.Queue(QUEUE.SYSTEM, baseQueueOptions(config, QUEUE.SYSTEM))), inject: [DI.config], }; const $endedPollNotification: Provider = { provide: 'queue:endedPollNotification', - useFactory: (config: Config) => new Bull.Queue(QUEUE.ENDED_POLL_NOTIFICATION, baseQueueOptions(config, QUEUE.ENDED_POLL_NOTIFICATION)), + useFactory: (config: Config) => withMetrics(new Bull.Queue(QUEUE.ENDED_POLL_NOTIFICATION, baseQueueOptions(config, QUEUE.ENDED_POLL_NOTIFICATION))), inject: [DI.config], }; const $deliver: Provider = { provide: 'queue:deliver', - useFactory: (config: Config) => new Bull.Queue(QUEUE.DELIVER, baseQueueOptions(config, QUEUE.DELIVER)), + useFactory: (config: Config) => withMetrics(new Bull.Queue(QUEUE.DELIVER, baseQueueOptions(config, QUEUE.DELIVER))), inject: [DI.config], }; const $inbox: Provider = { provide: 'queue:inbox', - useFactory: (config: Config) => new Bull.Queue(QUEUE.INBOX, baseQueueOptions(config, QUEUE.INBOX)), + useFactory: (config: Config) => withMetrics(new Bull.Queue(QUEUE.INBOX, baseQueueOptions(config, QUEUE.INBOX))), inject: [DI.config], }; const $db: Provider = { provide: 'queue:db', - useFactory: (config: Config) => new Bull.Queue(QUEUE.DB, baseQueueOptions(config, QUEUE.DB)), + useFactory: (config: Config) => withMetrics(new Bull.Queue(QUEUE.DB, baseQueueOptions(config, QUEUE.DB))), inject: [DI.config], }; const $relationship: Provider = { provide: 'queue:relationship', - useFactory: (config: Config) => new Bull.Queue(QUEUE.RELATIONSHIP, baseQueueOptions(config, QUEUE.RELATIONSHIP)), + useFactory: (config: Config) => withMetrics(new Bull.Queue(QUEUE.RELATIONSHIP, baseQueueOptions(config, QUEUE.RELATIONSHIP))), inject: [DI.config], }; const $objectStorage: Provider = { provide: 'queue:objectStorage', - useFactory: (config: Config) => new Bull.Queue(QUEUE.OBJECT_STORAGE, baseQueueOptions(config, QUEUE.OBJECT_STORAGE)), + useFactory: (config: Config) => withMetrics(new Bull.Queue(QUEUE.OBJECT_STORAGE, baseQueueOptions(config, QUEUE.OBJECT_STORAGE))), inject: [DI.config], }; const $userWebhookDeliver: Provider = { provide: 'queue:userWebhookDeliver', - useFactory: (config: Config) => new Bull.Queue(QUEUE.USER_WEBHOOK_DELIVER, baseQueueOptions(config, QUEUE.USER_WEBHOOK_DELIVER)), + useFactory: (config: Config) => withMetrics(new Bull.Queue(QUEUE.USER_WEBHOOK_DELIVER, baseQueueOptions(config, QUEUE.USER_WEBHOOK_DELIVER))), inject: [DI.config], }; const $systemWebhookDeliver: Provider = { provide: 'queue:systemWebhookDeliver', - useFactory: (config: Config) => new Bull.Queue(QUEUE.SYSTEM_WEBHOOK_DELIVER, baseQueueOptions(config, QUEUE.SYSTEM_WEBHOOK_DELIVER)), + useFactory: (config: Config) => withMetrics(new Bull.Queue(QUEUE.SYSTEM_WEBHOOK_DELIVER, baseQueueOptions(config, QUEUE.SYSTEM_WEBHOOK_DELIVER))), inject: [DI.config], }; diff --git a/packages/backend/src/core/activitypub/ApInboxService.ts b/packages/backend/src/core/activitypub/ApInboxService.ts index 376c9c0151..a3bd5678d4 100644 --- a/packages/backend/src/core/activitypub/ApInboxService.ts +++ b/packages/backend/src/core/activitypub/ApInboxService.ts @@ -5,6 +5,7 @@ import { Inject, Injectable } from '@nestjs/common'; import { In } from 'typeorm'; +import * as prom from 'prom-client'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import { UserFollowingService } from '@/core/UserFollowingService.js'; @@ -43,6 +44,12 @@ import type { IAccept, IAdd, IAnnounce, IBlock, ICreate, IDelete, IFlag, IFollow export class ApInboxService { private logger: Logger; + private mInboxReceived = new prom.Counter({ + name: 'misskey_ap_inbox_received_total', + help: 'Total number of activities received by AP inbox', + labelNames: ['host', 'type'], + }); + constructor( @Inject(DI.config) private config: Config, @@ -131,34 +138,49 @@ export class ApInboxService { if (actor.isSuspended) return; if (isCreate(activity)) { + this.mInboxReceived.inc({ host: actor.host, type: 'create' }); return await this.create(actor, activity); } else if (isDelete(activity)) { + this.mInboxReceived.inc({ host: actor.host, type: 'delete' }); return await this.delete(actor, activity); } else if (isUpdate(activity)) { + this.mInboxReceived.inc({ host: actor.host, type: 'update' }); return await this.update(actor, activity); } else if (isFollow(activity)) { + this.mInboxReceived.inc({ host: actor.host, type: 'follow' }); return await this.follow(actor, activity); } else if (isAccept(activity)) { + this.mInboxReceived.inc({ host: actor.host, type: 'accept' }); return await this.accept(actor, activity); } else if (isReject(activity)) { + this.mInboxReceived.inc({ host: actor.host, type: 'reject' }); return await this.reject(actor, activity); } else if (isAdd(activity)) { + this.mInboxReceived.inc({ host: actor.host, type: 'add' }); return await this.add(actor, activity); } else if (isRemove(activity)) { + this.mInboxReceived.inc({ host: actor.host, type: 'remove' }); return await this.remove(actor, activity); } else if (isAnnounce(activity)) { + this.mInboxReceived.inc({ host: actor.host, type: 'announce' }); return await this.announce(actor, activity); } else if (isLike(activity)) { + this.mInboxReceived.inc({ host: actor.host, type: 'like' }); return await this.like(actor, activity); } else if (isUndo(activity)) { + this.mInboxReceived.inc({ host: actor.host, type: 'undo' }); return await this.undo(actor, activity); } else if (isBlock(activity)) { + this.mInboxReceived.inc({ host: actor.host, type: 'block' }); return await this.block(actor, activity); } else if (isFlag(activity)) { + this.mInboxReceived.inc({ host: actor.host, type: 'flag' }); return await this.flag(actor, activity); } else if (isMove(activity)) { + this.mInboxReceived.inc({ host: actor.host, type: 'move' }); return await this.move(actor, activity); } else { + this.mInboxReceived.inc({ host: actor.host, type: 'unknown' }); return `unrecognized activity type: ${activity.type}`; } } diff --git a/packages/backend/src/misc/log-sanitization.ts b/packages/backend/src/misc/log-sanitization.ts new file mode 100644 index 0000000000..201de59528 --- /dev/null +++ b/packages/backend/src/misc/log-sanitization.ts @@ -0,0 +1,42 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project and yumechi + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { aidRegExp } from "./id/aid.js"; +import { aidxRegExp } from "./id/aidx.js"; + +export function sanitizeRequestURI(uri: string): string { + const vite = /^\/vite\/.+\.([a-z0-9]{1,4})$/; + const embed_vite = /^\/embed_vite\/.+\.([a-z0-9]{1,4})$/; + + if (vite.test(uri)) { + return '[vite]'; + } + + if (embed_vite.test(uri)) { + return '[embed_vite]'; + } + + const uuid = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/g; + const username_local = /\/@\w+(\/|$)/; + const username_remote = /\/@\w+@[a-zA-Z0-9-.]+\.[a-zA-Z]{2,4}(\/|$)/; + const token = /=[0-9a-zA-Z]{16}/g; + const aidx = new RegExp(`/${aidxRegExp.source.replace(/^\^/, '').replace(/\$$/, '')}(\/|$)`, 'g'); + const aid = new RegExp(`/${aidRegExp.source.replace(/^\^/, '').replace(/\$$/, '')}(\/|$)`, 'g'); + + return uri + .replace(/\/tags\/[^/]+/g, '/tags/[tag]') + .replace(/\/user-tags\/[^/]+/g, '/user-tags/[tag]') + .replace(/\/page\/[\w-]+/g, '/page/[page]') + .replace(/\/fluent-emoji\/[^/]+/g, '/fluent-emoji/[fluent-emoji]') + .replace(/\/twemoji\/[^/]+/g, '/twemoji/[twemoji]') + .replace(/\/twemoji-badge\/[^/]+/g, '/twemoji-badge/[twemoji-badge]') + .replace(aidx, '/[aidx]/') + .replace(aid, '/[aid]/') + .replace(token, '=[token]') + .replace(uuid, '[uuid]') + .replace(username_local, '/[user_local]/') + .replace(username_remote, '/[user_remote]/'); +} + diff --git a/packages/backend/src/postgres.ts b/packages/backend/src/postgres.ts index 870f5ff36e..6fce6a6faa 100644 --- a/packages/backend/src/postgres.ts +++ b/packages/backend/src/postgres.ts @@ -7,6 +7,8 @@ import pg from 'pg'; import { DataSource, Logger } from 'typeorm'; import * as highlight from 'cli-highlight'; +import * as prom from 'prom-client'; +import { createHash } from 'crypto'; import { entities as charts } from '@/core/chart/entities.js'; import { MiAbuseUserReport } from '@/models/AbuseUserReport.js'; @@ -82,6 +84,7 @@ import { MiReversiGame } from '@/models/ReversiGame.js'; import { Config } from '@/config.js'; import MisskeyLogger from '@/logger.js'; import { bindThis } from '@/decorators.js'; +import { MemoryKVCache } from './misc/cache.js'; pg.types.setTypeParser(20, Number); @@ -89,7 +92,63 @@ export const dbLogger = new MisskeyLogger('db'); const sqlLogger = dbLogger.createSubLogger('sql', 'gray'); +type QueryTagCache = { + join: string; + from: string; + hash: string; +}; + +function dedupConsecutive(arr: T[]): T[] { + return arr.filter((v, i, a) => i === 0 || a[i - 1] !== v); +} + +function simplifyIdentifiers(sql: string) { + return sql.replace(/"([a-zA-Z_]+)"/g, '$1'); +} + +function extractQueryTags(query: string): QueryTagCache { + const joins = query.matchAll(/(LEFT|RIGHT|INNER|OUTER) JOIN ([a-zA-Z0-9_"`.\s]+) ON/ig); + const froms = query.matchAll(/FROM ([a-zA-Z0-9_"`.]+)/ig); + + const join = Array.from(joins).map(j => `${j[1]}:${simplifyIdentifiers(j[2])}`).join('|'); + const from = dedupConsecutive(Array.from(froms).map(f => simplifyIdentifiers(f[1]))).join('|'); + + // this is not for security just to reduce metrics size just for confirmation in local environment + // all the code is public there is nothing secret in the query itself + const hash = createHash('md5'); + hash.update(query); + + return { + join, + from, + hash: hash.digest('hex'), + }; +} + +const mQueryCounter = new prom.Counter({ + name: 'misskey_postgres_query_total', + help: 'Total queries to postgres', + labelNames: ['join', 'from', 'hash'], +}); + +const mQueryErrorCounter = new prom.Counter({ + name: 'misskey_postgres_query_error_total', + help: 'Total errors in queries to postgres', + labelNames: ['join', 'from', 'hash'], +}); + +const mSlowQueryHisto = new prom.Histogram({ + name: 'misskey_postgres_query_slow_duration_seconds', + help: 'Duration of slow queries to postgres', + labelNames: ['join', 'from', 'hash'], + buckets: [0.1, 0.5, 1, 2, 5, 10, 30, 300], +}); + class MyCustomLogger implements Logger { + constructor(private metricOnly = true) {} + + private queryHashCache = new MemoryKVCache(1000 * 60 * 5); // 5m + @bindThis private highlight(sql: string) { return highlight.highlight(sql, { @@ -97,33 +156,69 @@ class MyCustomLogger implements Logger { }); } + @bindThis + private getQueryTags(query: string): QueryTagCache { + const existing = this.queryHashCache.get(query); + if (existing) { + return existing; + } + + const result = extractQueryTags(query); + this.queryHashCache.set(query, result); + return result; + } + @bindThis public logQuery(query: string, parameters?: any[]) { + mQueryCounter.inc(this.getQueryTags(query)); + + if (this.metricOnly) { + return; + } sqlLogger.info(this.highlight(query).substring(0, 100)); } @bindThis public logQueryError(error: string, query: string, parameters?: any[]) { + mQueryErrorCounter.inc(this.getQueryTags(query)); + + if (this.metricOnly) { + return; + } sqlLogger.error(this.highlight(query)); } @bindThis public logQuerySlow(time: number, query: string, parameters?: any[]) { + mSlowQueryHisto.observe(this.getQueryTags(query), time); + + if (this.metricOnly) { + return; + } sqlLogger.warn(this.highlight(query)); } @bindThis public logSchemaBuild(message: string) { + if (this.metricOnly) { + return; + } sqlLogger.info(message); } @bindThis public log(message: string) { + if (this.metricOnly) { + return; + } sqlLogger.info(message); } @bindThis public logMigration(message: string) { + if (this.metricOnly) { + return; + } sqlLogger.info(message); } } @@ -203,7 +298,7 @@ export const entities = [ const log = process.env.NODE_ENV !== 'production'; -export function createPostgresDataSource(config: Config) { +export function createPostgresDataSource(config: Config, isMain = false) { return new DataSource({ type: 'postgres', host: config.db.host, @@ -246,9 +341,9 @@ export function createPostgresDataSource(config: Config) { db: config.redis.db ?? 0, }, } : false, - logging: log, - logger: log ? new MyCustomLogger() : undefined, - maxQueryExecutionTime: 1000, + logging: log ? 'all' : ['query'], + logger: (isMain || log) ? new MyCustomLogger(!log) : undefined, + maxQueryExecutionTime: 500, entities: entities, migrations: ['../../migration/*.js'], }); diff --git a/packages/backend/src/queue/QueueProcessorService.ts b/packages/backend/src/queue/QueueProcessorService.ts index 6940e1c188..9a6b3d8063 100644 --- a/packages/backend/src/queue/QueueProcessorService.ts +++ b/packages/backend/src/queue/QueueProcessorService.ts @@ -45,6 +45,7 @@ import { CleanProcessorService } from './processors/CleanProcessorService.js'; import { AggregateRetentionProcessorService } from './processors/AggregateRetentionProcessorService.js'; import { QueueLoggerService } from './QueueLoggerService.js'; import { QUEUE, baseQueueOptions } from './const.js'; +import { mStalledWorkerCounter } from './metrics.js'; // ref. https://github.com/misskey-dev/misskey/pull/7635#issue-971097019 function httpRelatedBackoff(attemptsMade: number) { @@ -194,7 +195,10 @@ export class QueueProcessorService implements OnApplicationShutdown { } }) .on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) })) - .on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`)); + .on('stalled', (jobId) => { + mStalledWorkerCounter.inc({ queue: QUEUE.SYSTEM }); + logger.warn(`stalled id=${jobId}`); + }); } //#endregion @@ -251,7 +255,10 @@ export class QueueProcessorService implements OnApplicationShutdown { } }) .on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) })) - .on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`)); + .on('stalled', (jobId) => { + mStalledWorkerCounter.inc({ queue: QUEUE.DB }); + logger.warn(`stalled id=${jobId}`); + }); } //#endregion @@ -291,7 +298,10 @@ export class QueueProcessorService implements OnApplicationShutdown { } }) .on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) })) - .on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`)); + .on('stalled', (jobId) => { + mStalledWorkerCounter.inc({ queue: QUEUE.DELIVER }); + logger.warn(`stalled id=${jobId}`); + }); } //#endregion @@ -331,7 +341,10 @@ export class QueueProcessorService implements OnApplicationShutdown { } }) .on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) })) - .on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`)); + .on('stalled', (jobId) => { + mStalledWorkerCounter.inc({ queue: QUEUE.INBOX }); + logger.warn(`stalled id=${jobId}`); + }); } //#endregion @@ -371,7 +384,10 @@ export class QueueProcessorService implements OnApplicationShutdown { } }) .on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) })) - .on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`)); + .on('stalled', (jobId) => { + mStalledWorkerCounter.inc({ queue: QUEUE.USER_WEBHOOK_DELIVER }); + logger.warn(`stalled id=${jobId}`); + }); } //#endregion @@ -411,7 +427,10 @@ export class QueueProcessorService implements OnApplicationShutdown { } }) .on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) })) - .on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`)); + .on('stalled', (jobId) => { + mStalledWorkerCounter.inc({ queue: QUEUE.SYSTEM_WEBHOOK_DELIVER }); + logger.warn(`stalled id=${jobId}`); + }); } //#endregion @@ -458,7 +477,10 @@ export class QueueProcessorService implements OnApplicationShutdown { } }) .on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) })) - .on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`)); + .on('stalled', (jobId) => { + mStalledWorkerCounter.inc({ queue: QUEUE.RELATIONSHIP }); + logger.warn(`stalled id=${jobId}`); + }); } //#endregion @@ -499,7 +521,10 @@ export class QueueProcessorService implements OnApplicationShutdown { } }) .on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) })) - .on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`)); + .on('stalled', (jobId) => { + mStalledWorkerCounter.inc({ queue: QUEUE.OBJECT_STORAGE }); + logger.warn(`stalled id=${jobId}`); + }); } //#endregion diff --git a/packages/backend/src/queue/metrics.ts b/packages/backend/src/queue/metrics.ts new file mode 100644 index 0000000000..a3e794d6a9 --- /dev/null +++ b/packages/backend/src/queue/metrics.ts @@ -0,0 +1,31 @@ +import * as prom from 'prom-client'; + +export const mJobReceivedCounter = new prom.Counter({ + name: 'misskey_queue_jobs_received_total', + help: 'Total number of jobs received by queue', + labelNames: ['queue'], +}); + +export const mActiveJobs = new prom.Gauge({ + name: 'misskey_queue_active_jobs', + help: 'Number of active jobs in queue', + labelNames: ['queue'], +}); + +export const mDelayedJobs = new prom.Gauge({ + name: 'misskey_queue_delayed_jobs', + help: 'Number of delayed jobs in queue', + labelNames: ['queue'], +}); + +export const mWaitingJobs = new prom.Gauge({ + name: 'misskey_queue_waiting_jobs', + help: 'Number of waiting jobs in queue', + labelNames: ['queue'], +}); + +export const mStalledWorkerCounter = new prom.Counter({ + name: 'misskey_queue_stalled_workers_total', + help: 'Total number of stalled workers', + labelNames: ['queue'], +}); diff --git a/packages/backend/src/queue/processors/InboxProcessorService.ts b/packages/backend/src/queue/processors/InboxProcessorService.ts index 95d764e4d8..bd6da6c668 100644 --- a/packages/backend/src/queue/processors/InboxProcessorService.ts +++ b/packages/backend/src/queue/processors/InboxProcessorService.ts @@ -7,6 +7,7 @@ import { URL } from 'node:url'; import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import httpSignature from '@peertube/http-signature'; import * as Bull from 'bullmq'; +import * as prom from 'prom-client'; import type Logger from '@/logger.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; import { FetchInstanceMetadataService } from '@/core/FetchInstanceMetadataService.js'; @@ -42,6 +43,37 @@ export class InboxProcessorService implements OnApplicationShutdown { private logger: Logger; private updateInstanceQueue: CollapsedQueue; + private mIncomingApProcessingTime = new prom.Histogram({ + name: 'misskey_incoming_ap_processing_time', + help: 'Incoming AP processing time in seconds', + labelNames: ['incoming_host', 'incoming_type', 'success'], + buckets: [0.01, 0.1, 0.5, 1, 5, 10, 30, 60, 300, 1800], + }); + + private mIncomingApEvent = new prom.Counter({ + name: 'misskey_incoming_ap_event', + help: 'Incoming AP event', + labelNames: ['incoming_host', 'incoming_type'], + }); + + private mIncomingApEventAccepted = new prom.Counter({ + name: 'misskey_incoming_ap_event_accepted', + help: 'Incoming AP event accepted', + labelNames: ['incoming_host', 'incoming_type'], + }); + + private mIncomingApReject = new prom.Counter({ + name: 'misskey_incoming_ap_reject', + help: 'Incoming AP reject', + labelNames: ['incoming_host', 'incoming_type', 'reason'], + }); + + private mincomingApProcessingError = new prom.Counter({ + name: 'misskey_incoming_ap_processing_error', + help: 'Incoming AP processing error', + labelNames: ['incoming_host', 'incoming_type'], + }); + constructor( @Inject(DI.meta) private meta: MiMeta, @@ -66,7 +98,6 @@ export class InboxProcessorService implements OnApplicationShutdown { public async process(job: Bull.Job): Promise { const signature = job.data.signature; // HTTP-signature let activity = job.data.activity; - //#region Log const info = Object.assign({}, activity); delete info['@context']; @@ -75,12 +106,34 @@ export class InboxProcessorService implements OnApplicationShutdown { const host = this.utilityService.toPuny(new URL(signature.keyId).hostname); + const incCounter = (counter: prom.Counter, addn_labels: U) => { + if (Array.isArray(activity.type)) { + for (const t of activity.type) { + counter.inc({ incoming_host: host.toString(), incoming_type: t, ...addn_labels }); + } + } else { + counter.inc({ incoming_host: host.toString(), incoming_type: activity.type ?? 'unknown', ...addn_labels }); + } + }; + + const observeHistogram = (histogram: prom.Histogram, addn_labels: U, value: number) => { + if (Array.isArray(activity.type)) { + for (const t of activity.type) { + histogram.observe({ incoming_host: host.toString(), incoming_type: t, ...addn_labels }, value); + } + } else { + histogram.observe({ incoming_host: host.toString(), incoming_type: activity.type ?? 'unknown', ...addn_labels }, value); + } + }; + if (!this.utilityService.isFederationAllowedHost(host)) { + incCounter(this.mIncomingApReject, { reason: 'host_not_allowed' }); return `Blocked request: ${host}`; } const keyIdLower = signature.keyId.toLowerCase(); if (keyIdLower.startsWith('acct:')) { + incCounter(this.mIncomingApReject, { reason: 'keyid_acct' }); return `Old keyId is no longer supported. ${keyIdLower}`; } @@ -98,6 +151,7 @@ export class InboxProcessorService implements OnApplicationShutdown { // 対象が4xxならスキップ if (err instanceof StatusError) { if (!err.isRetryable) { + incCounter(this.mIncomingApReject, { reason: 'actor_key_unresolvable' }); throw new Bull.UnrecoverableError(`skip: Ignored deleted actors on both ends ${activity.actor} - ${err.statusCode}`); } throw new Error(`Error in actor ${activity.actor} - ${err.statusCode}`); @@ -107,11 +161,13 @@ export class InboxProcessorService implements OnApplicationShutdown { // それでもわからなければ終了 if (authUser == null) { + incCounter(this.mIncomingApReject, { reason: 'actor_unresolvable' }); throw new Bull.UnrecoverableError('skip: failed to resolve user'); } // publicKey がなくても終了 if (authUser.key == null) { + incCounter(this.mIncomingApReject, { reason: 'publickey_unresolvable' }); throw new Bull.UnrecoverableError('skip: failed to resolve user publicKey'); } @@ -124,6 +180,7 @@ export class InboxProcessorService implements OnApplicationShutdown { const ldSignature = activity.signature; if (ldSignature) { if (ldSignature.type !== 'RsaSignature2017') { + incCounter(this.mIncomingApReject, { reason: 'ld_signature_unsupported' }); throw new Bull.UnrecoverableError(`skip: unsupported LD-signature type ${ldSignature.type}`); } @@ -137,10 +194,12 @@ export class InboxProcessorService implements OnApplicationShutdown { // keyIdからLD-Signatureのユーザーを取得 authUser = await this.apDbResolverService.getAuthUserFromKeyId(ldSignature.creator); if (authUser == null) { + incCounter(this.mIncomingApReject, { reason: 'ld_signature_user_unresolvable' }); throw new Bull.UnrecoverableError('skip: LD-Signatureのユーザーが取得できませんでした'); } if (authUser.key == null) { + incCounter(this.mIncomingApReject, { reason: 'ld_signature_publickey_unavailable' }); throw new Bull.UnrecoverableError('skip: LD-SignatureのユーザーはpublicKeyを持っていませんでした'); } @@ -149,6 +208,7 @@ export class InboxProcessorService implements OnApplicationShutdown { // LD-Signature検証 const verified = await jsonLd.verifyRsaSignature2017(activity, authUser.key.keyPem).catch(() => false); if (!verified) { + incCounter(this.mIncomingApReject, { reason: 'ld_signature_verification_failed' }); throw new Bull.UnrecoverableError('skip: LD-Signatureの検証に失敗しました'); } @@ -171,14 +231,17 @@ export class InboxProcessorService implements OnApplicationShutdown { // もう一度actorチェック if (authUser.user.uri !== activity.actor) { + incCounter(this.mIncomingApReject, { reason: 'ld_signature_actor_mismatch' }); throw new Bull.UnrecoverableError(`skip: LD-Signature user(${authUser.user.uri}) !== activity.actor(${activity.actor})`); } const ldHost = this.utilityService.extractDbHost(authUser.user.uri); if (!this.utilityService.isFederationAllowedHost(ldHost)) { + incCounter(this.mIncomingApReject, { reason: 'fed_host_not_allowed' }); throw new Bull.UnrecoverableError(`Blocked request: ${ldHost}`); } } else { + incCounter(this.mIncomingApReject, { reason: 'ld_signature_unavailable' }); throw new Bull.UnrecoverableError(`skip: http-signature verification failed and no LD-Signature. keyId=${signature.keyId}`); } } @@ -188,6 +251,7 @@ export class InboxProcessorService implements OnApplicationShutdown { const signerHost = this.utilityService.extractDbHost(authUser.user.uri!); const activityIdHost = this.utilityService.extractDbHost(activity.id); if (signerHost !== activityIdHost) { + incCounter(this.mIncomingApReject, 'host_signature_mismatch'); throw new Bull.UnrecoverableError(`skip: signerHost(${signerHost}) !== activity.id host(${activityIdHost}`); } } @@ -215,7 +279,10 @@ export class InboxProcessorService implements OnApplicationShutdown { this.fetchInstanceMetadataService.fetchInstanceMetadata(i); }); + incCounter(this.mIncomingApEvent, {}); + // アクティビティを処理 + const begin = +new Date(); try { const result = await this.apInboxService.performActivity(authUser.user, activity); if (result && !result.startsWith('ok')) { @@ -225,17 +292,26 @@ export class InboxProcessorService implements OnApplicationShutdown { } catch (e) { if (e instanceof IdentifiableError) { if (e.id === '689ee33f-f97c-479a-ac49-1b9f8140af99') { + incCounter(this.mIncomingApReject, { reason: 'blocked_notes_with_prohibited_words' }); return 'blocked notes with prohibited words'; } if (e.id === '85ab9bd7-3a41-4530-959d-f07073900109') { + incCounter(this.mIncomingApReject, { reason: 'actor_suspended' }); return 'actor has been suspended'; } if (e.id === 'd450b8a9-48e4-4dab-ae36-f4db763fda7c') { // invalid Note + incCounter(this.mIncomingApReject, { reason: 'invalid_note' }); return e.message; } } + const end = +new Date(); + observeHistogram(this.mIncomingApProcessingTime, { success: 'false' }, (end - begin) / 1000); + incCounter(this.mincomingApProcessingError, { reason: 'unknown' }); throw e; } + observeHistogram(this.mIncomingApProcessingTime, { success: 'true' }, (+new Date() - begin) / 1000); + incCounter(this.mIncomingApEventAccepted, {}); + return 'ok'; } diff --git a/packages/backend/src/server/ServerModule.ts b/packages/backend/src/server/ServerModule.ts index 3ab0b815f2..df1d0c0d54 100644 --- a/packages/backend/src/server/ServerModule.ts +++ b/packages/backend/src/server/ServerModule.ts @@ -47,6 +47,7 @@ import { RoleTimelineChannelService } from './api/stream/channels/role-timeline. import { ReversiChannelService } from './api/stream/channels/reversi.js'; import { ReversiGameChannelService } from './api/stream/channels/reversi-game.js'; import { SigninWithPasskeyApiService } from './api/SigninWithPasskeyApiService.js'; +import { MetricsService } from './api/MetricsService.js'; @Module({ imports: [ @@ -94,6 +95,7 @@ import { SigninWithPasskeyApiService } from './api/SigninWithPasskeyApiService.j UserListChannelService, OpenApiServerService, OAuth2ProviderService, + MetricsService, ], exports: [ ServerService, diff --git a/packages/backend/src/server/ServerService.ts b/packages/backend/src/server/ServerService.ts index be02274600..bbb0503c1e 100644 --- a/packages/backend/src/server/ServerService.ts +++ b/packages/backend/src/server/ServerService.ts @@ -33,14 +33,88 @@ import { OpenApiServerService } from './api/openapi/OpenApiServerService.js'; import { OAuth2ProviderService } from './oauth/OAuth2ProviderService.js'; import { makeHstsHook } from './hsts.js'; import { generateCSP } from './csp.js'; +import * as prom from 'prom-client'; +import { sanitizeRequestURI } from '@/misc/log-sanitization.js'; +import { MetricsService } from './api/MetricsService.js'; const _dirname = fileURLToPath(new URL('.', import.meta.url)); +function categorizeRequestPath(path: string): 'api' | 'health' | 'vite' | 'other' { + if (path === '/healthz') { + return 'health'; + } + + if (path.startsWith('/vite/') || path.startsWith('/embed_vite/')) { + return 'vite'; + } + + if (path === '/api' || path.startsWith('/api/')) { + return 'api'; + } + + return 'other'; +} + @Injectable() export class ServerService implements OnApplicationShutdown { private logger: Logger; #fastify: FastifyInstance; + private mRequestTime = new prom.Histogram({ + name: 'misskey_http_request_duration_seconds', + help: 'Duration of handling HTTP requests in seconds', + labelNames: ['host', 'cate', 'method', 'path'], + buckets: [0.001, 0.1, 0.5, 1, 2, 5], + }); + + private mRequestsReceived = new prom.Counter({ + name: 'misskey_http_requests_received_total', + help: 'Total number of HTTP requests received', + labelNames: [], + }); + + private mNotFoundServed = new prom.Counter({ + name: 'misskey_http_not_found_served_total', + help: 'Total number of HTTP 404 responses served', + labelNames: ['method', 'cate'], + }); + + private mMethodNotAllowedServed = new prom.Counter({ + name: 'misskey_http_method_not_allowed_served_total', + help: 'Total number of HTTP 405 responses served', + labelNames: ['method', 'cate'], + }); + + private mTooManyRequestsServed = new prom.Counter({ + name: 'misskey_http_too_many_requests_served_total', + help: 'Total number of HTTP 429 responses served', + labelNames: ['method', 'cate'], + }); + + private mAggregateRequestsServed = new prom.Counter({ + name: 'misskey_http_requests_served_total', + help: 'Total number of HTTP requests served including invalid requests', + labelNames: ['host', 'cate', 'status'], + }); + + private mRequestsServedByPath = new prom.Counter({ + name: 'misskey_http_requests_served_by_path', + help: 'Total number of HTTP requests served', + labelNames: ['host', 'cate', 'method', 'path', 'status'], + }); + + private mFatalErrorCount = new prom.Counter({ + name: 'misskey_fatal_http_errors_total', + help: 'Total number of HTTP errors that propagate to the top level', + labelNames: ['host', 'cate', 'method', 'path'], + }); + + private mLastSuccessfulRequest = new prom.Gauge({ + name: 'misskey_http_last_successful_request_timestamp_seconds', + help: 'Unix Timestamp of the last successful HTTP request', + labelNames: [], + }); + constructor( @Inject(DI.config) private config: Config, @@ -70,6 +144,7 @@ export class ServerService implements OnApplicationShutdown { private globalEventService: GlobalEventService, private loggerService: LoggerService, private oauth2ProviderService: OAuth2ProviderService, + private metricsService: MetricsService, ) { this.logger = this.loggerService.getLogger('server', 'gray'); } @@ -82,6 +157,111 @@ export class ServerService implements OnApplicationShutdown { }); this.#fastify = fastify; + if (this.config.prometheusMetrics?.enable) { + fastify.addHook('onRequest', (_request, reply, done) => { + reply.header('x-request-received', (+new Date()).toString()); + this.mRequestsReceived.inc(); + done(); + }); + + fastify.addHook('onError', (request, _reply, error, done) => { + const url = new URL(request.url, this.config.url); + const logPath = sanitizeRequestURI(url.pathname); + this.mFatalErrorCount.inc({ + host: request.hostname, + method: request.method, + path: logPath, + cate: categorizeRequestPath(logPath), + }); + done(); + }); + + fastify.addHook('onResponse', (request, reply, done) => { + const url = new URL(request.url, this.config.url); + const logPath = sanitizeRequestURI(url.pathname); + const cate = categorizeRequestPath(logPath); + const received = reply.getHeader('x-request-received') as string; + + this.mAggregateRequestsServed.inc({ + host: request.hostname, + cate, + status: reply.statusCode, + }); + + if (reply.statusCode === 429) { + this.mTooManyRequestsServed.inc({ + method: request.method, + cate, + }); + + done(); + return; + } + + if (reply.statusCode === 404) { + this.mNotFoundServed.inc({ + method: request.method, + cate, + }); + + if (received) { + const duration = (+new Date()) - parseInt(received); + this.mRequestTime.observe({ + host: request.hostname, + method: request.method, + cate, + }, duration / 1000); + } + + done(); + return; + } + + if (reply.statusCode === 405) { + this.mMethodNotAllowedServed.inc({ + method: request.method, + cate, + }); + done(); + return; + } + + if (received) { + const duration = (+new Date()) - parseInt(received); + + this.mRequestTime.observe({ + host: request.hostname, + method: request.method, + cate, + path: logPath, + }, duration / 1000); + } + + if (logPath === '/metrics' || logPath === '/healthz') { + done(); + return; + } + + if (reply.statusCode <= 299) { + this.mLastSuccessfulRequest.set(+new Date() / 1000); + } + + this.mRequestsServedByPath.inc({ + host: request.hostname, + method: request.method, + path: logPath, + cate, + status: reply.statusCode >= 500 ? '5xx' : + reply.statusCode === 401 ? '401' : + reply.statusCode === 403 ? '403' : + reply.statusCode >= 400 ? + '4xx' : '2xx', + }); + + done(); + }); + } + // HSTS if (this.config.url.startsWith('https') && !this.config.disableHsts) { const preload = this.config.hstsPreload; @@ -253,6 +433,8 @@ export class ServerService implements OnApplicationShutdown { fastify.register(this.clientServerService.createServer); + fastify.register(this.metricsService.createServer); + this.streamingApiServerService.attach(fastify.server); fastify.server.on('error', err => { diff --git a/packages/backend/src/server/api/AuthenticateService.ts b/packages/backend/src/server/api/AuthenticateService.ts index 690ff2e022..13a51e3797 100644 --- a/packages/backend/src/server/api/AuthenticateService.ts +++ b/packages/backend/src/server/api/AuthenticateService.ts @@ -5,6 +5,7 @@ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import { DI } from '@/di-symbols.js'; +import * as prom from 'prom-client'; import type { AccessTokensRepository, AppsRepository, UsersRepository } from '@/models/_.js'; import type { MiLocalUser } from '@/models/User.js'; import type { MiAccessToken } from '@/models/AccessToken.js'; @@ -25,6 +26,12 @@ export class AuthenticationError extends Error { export class AuthenticateService implements OnApplicationShutdown { private appCache: MemoryKVCache; + private mAuthenticationFailureCounter = new prom.Counter({ + name: 'misskey_authentication_failure_total', + help: 'Total number of authentication failures', + labelNames: ['cred_ty'], + }); + constructor( @Inject(DI.usersRepository) private usersRepository: UsersRepository, @@ -51,6 +58,7 @@ export class AuthenticateService implements OnApplicationShutdown { () => this.usersRepository.findOneBy({ token }) as Promise); if (user == null) { + this.mAuthenticationFailureCounter.inc({ cred_ty: 'native' }); throw new AuthenticationError('user not found'); } @@ -65,6 +73,7 @@ export class AuthenticateService implements OnApplicationShutdown { }); if (accessToken == null) { + this.mAuthenticationFailureCounter.inc({ cred_ty: 'access_token' }); throw new AuthenticationError('invalid signature'); } diff --git a/packages/backend/src/server/api/MetricsService.ts b/packages/backend/src/server/api/MetricsService.ts new file mode 100644 index 0000000000..034098960d --- /dev/null +++ b/packages/backend/src/server/api/MetricsService.ts @@ -0,0 +1,52 @@ +import { Inject, Injectable } from "@nestjs/common"; +import * as prom from 'prom-client'; +import { DI } from '@/di-symbols.js'; +import type { Config } from '@/config.js'; +import { bindThis } from "@/decorators.js"; +import type { FastifyInstance, FastifyPluginOptions } from "fastify"; + +/* + * SPDX-FileCopyrightText: syuilo and misskey-project and yumechi + * SPDX-License-Identifier: AGPL-3.0-only + */ + +@Injectable() +export class MetricsService { + constructor( + @Inject(DI.config) + private config: Config + ) {} + + @bindThis + public createServer(fastify: FastifyInstance, options: FastifyPluginOptions, done: (err?: Error) => void) { + if (this.config.prometheusMetrics?.enable) { + const token = this.config.prometheusMetrics.scrapeToken; + fastify.get('/metrics', async (request, reply) => { + if (token) { + const bearer = request.headers.authorization; + + if (!bearer) { + reply.code(401); + return; + } + + const [type, t] = bearer.split(' '); + + if (type !== 'Bearer' || t !== token) { + reply.code(403); + return; + } + } + + try { + reply.header('Content-Type', prom.register.contentType); + reply.send(await prom.register.metrics()); + } catch (err) { + reply.code(500); + } + }); + } + + done(); + } +} diff --git a/packages/backend/src/server/api/SigninApiService.ts b/packages/backend/src/server/api/SigninApiService.ts index 1d983ca4bc..80cfdf5436 100644 --- a/packages/backend/src/server/api/SigninApiService.ts +++ b/packages/backend/src/server/api/SigninApiService.ts @@ -6,6 +6,7 @@ import { Inject, Injectable } from '@nestjs/common'; import bcrypt from 'bcryptjs'; import { IsNull } from 'typeorm'; +import * as prom from 'prom-client'; import * as Misskey from 'misskey-js'; import { DI } from '@/di-symbols.js'; import type { @@ -31,6 +32,12 @@ import type { FastifyReply, FastifyRequest } from 'fastify'; @Injectable() export class SigninApiService { + private mSigninFailureCounter = new prom.Counter({ + name: 'misskey_misskey_signin_failure', + help: 'The number of failed sign-ins', + labelNames: ['reason'], + }); + constructor( @Inject(DI.config) private config: Config, @@ -93,6 +100,7 @@ export class SigninApiService { // not more than 1 attempt per second and not more than 10 attempts per hour await this.rateLimiterService.limit({ key: 'signin', duration: 60 * 60 * 1000, max: 10, minInterval: 1000 }, getIpHash(request.ip)); } catch (err) { + this.mSigninFailureCounter.inc({ reason: 'rate_limit' }); reply.code(429); return { error: { @@ -104,11 +112,13 @@ export class SigninApiService { } if (typeof username !== 'string') { + this.mSigninFailureCounter.inc({ reason: 'bad_form' }); reply.code(400); return; } if (token != null && typeof token !== 'string') { + this.mSigninFailureCounter.inc({ reason: 'bad_form' }); reply.code(400); return; } @@ -120,12 +130,14 @@ export class SigninApiService { }) as MiLocalUser; if (user == null) { + this.mSigninFailureCounter.inc({ reason: 'user_not_found' }); return error(404, { id: '6cc579cc-885d-43d8-95c2-b8c7fc963280', }); } if (user.isSuspended) { + this.mSigninFailureCounter.inc({ reason: 'user_suspended' }); return error(403, { id: 'e03a5f46-d309-4865-9b69-56282d94e1eb', }); @@ -150,6 +162,7 @@ export class SigninApiService { } if (typeof password !== 'string') { + this.mSigninFailureCounter.inc({ reason: 'bad_form' }); reply.code(400); return; } @@ -167,6 +180,7 @@ export class SigninApiService { success: false, }); + this.mSigninFailureCounter.inc({ reason: failure?.id ?? `unknown_error_${status ?? 500}` }); return error(status ?? 500, failure ?? { id: '4e30e80c-e338-45a0-8c8f-44455efa3b76' }); }; @@ -174,30 +188,35 @@ export class SigninApiService { if (process.env.NODE_ENV !== 'test') { if (this.meta.enableHcaptcha && this.meta.hcaptchaSecretKey) { await this.captchaService.verifyHcaptcha(this.meta.hcaptchaSecretKey, body['hcaptcha-response']).catch(err => { + this.mSigninFailureCounter.inc({ reason: 'captcha_verification_failed_hcaptcha' }); throw new FastifyReplyError(400, err); }); } if (this.meta.enableMcaptcha && this.meta.mcaptchaSecretKey && this.meta.mcaptchaSitekey && this.meta.mcaptchaInstanceUrl) { await this.captchaService.verifyMcaptcha(this.meta.mcaptchaSecretKey, this.meta.mcaptchaSitekey, this.meta.mcaptchaInstanceUrl, body['m-captcha-response']).catch(err => { + this.mSigninFailureCounter.inc({ reason: 'captcha_verification_failed_mcaptcha' }); throw new FastifyReplyError(400, err); }); } if (this.meta.enableRecaptcha && this.meta.recaptchaSecretKey) { await this.captchaService.verifyRecaptcha(this.meta.recaptchaSecretKey, body['g-recaptcha-response']).catch(err => { + this.mSigninFailureCounter.inc({ reason: 'captcha_verification_failed_recaptcha' }); throw new FastifyReplyError(400, err); }); } if (this.meta.enableTurnstile && this.meta.turnstileSecretKey) { await this.captchaService.verifyTurnstile(this.meta.turnstileSecretKey, body['turnstile-response']).catch(err => { + this.mSigninFailureCounter.inc({ reason: 'captcha_verification_failed_turnstile' }); throw new FastifyReplyError(400, err); }); } if (this.meta.enableTestcaptcha) { await this.captchaService.verifyTestcaptcha(body['testcaptcha-response']).catch(err => { + this.mSigninFailureCounter.inc({ reason: 'captcha_verification_failed_testcaptcha' }); throw new FastifyReplyError(400, err); }); } diff --git a/packages/frontend/.storybook/fakes.ts b/packages/frontend/.storybook/fakes.ts index fc3b0334e4..43c01ba44d 100644 --- a/packages/frontend/.storybook/fakes.ts +++ b/packages/frontend/.storybook/fakes.ts @@ -167,7 +167,7 @@ export function federationInstance(): entities.FederationInstance { notesCount: 20, followingCount: 5, followersCount: 15, - isNotResponding: false, + : false, isSuspended: false, suspensionState: 'none', isBlocked: false, diff --git a/packages/misskey-js/etc/misskey-js.api.md b/packages/misskey-js/etc/misskey-js.api.md index e39b041319..e012e00263 100644 --- a/packages/misskey-js/etc/misskey-js.api.md +++ b/packages/misskey-js/etc/misskey-js.api.md @@ -1812,10 +1812,6 @@ declare namespace entities { MeDetailed, UserDetailed, User, - UserWebhookBody, - UserWebhookNoteBody, - UserWebhookUserBody, - UserWebhookReactionBody, UserList, Ad, Announcement, @@ -3414,18 +3410,6 @@ type UsersShowResponse = operations['users___show']['responses']['200']['content // @public (undocumented) type UsersUpdateMemoRequest = operations['users___update-memo']['requestBody']['content']['application/json']; -// @public (undocumented) -type UserWebhookBody = components['schemas']['UserWebhookBody']; - -// @public (undocumented) -type UserWebhookNoteBody = components['schemas']['UserWebhookNoteBody']; - -// @public (undocumented) -type UserWebhookReactionBody = components['schemas']['UserWebhookReactionBody']; - -// @public (undocumented) -type UserWebhookUserBody = components['schemas']['UserWebhookUserBody']; - // Warnings were encountered during analysis: // // src/entities.ts:50:2 - (ae-forgotten-export) The symbol "ModerationLogPayloads" needs to be exported by the entry point index.d.ts diff --git a/packages/misskey-js/src/autogen/models.ts b/packages/misskey-js/src/autogen/models.ts index 8b1711015c..04574849d4 100644 --- a/packages/misskey-js/src/autogen/models.ts +++ b/packages/misskey-js/src/autogen/models.ts @@ -7,10 +7,6 @@ export type UserDetailedNotMe = components['schemas']['UserDetailedNotMe']; export type MeDetailed = components['schemas']['MeDetailed']; export type UserDetailed = components['schemas']['UserDetailed']; export type User = components['schemas']['User']; -export type UserWebhookBody = components['schemas']['UserWebhookBody']; -export type UserWebhookNoteBody = components['schemas']['UserWebhookNoteBody']; -export type UserWebhookUserBody = components['schemas']['UserWebhookUserBody']; -export type UserWebhookReactionBody = components['schemas']['UserWebhookReactionBody']; export type UserList = components['schemas']['UserList']; export type Ad = components['schemas']['Ad']; export type Announcement = components['schemas']['Announcement']; diff --git a/packages/misskey-js/src/autogen/types.ts b/packages/misskey-js/src/autogen/types.ts index 8950a86ff1..280abba727 100644 --- a/packages/misskey-js/src/autogen/types.ts +++ b/packages/misskey-js/src/autogen/types.ts @@ -4022,32 +4022,6 @@ export type components = { MeDetailed: components['schemas']['UserLite'] & components['schemas']['UserDetailedNotMeOnly'] & components['schemas']['MeDetailedOnly']; UserDetailed: components['schemas']['UserDetailedNotMe'] | components['schemas']['MeDetailed']; User: components['schemas']['UserLite'] | components['schemas']['UserDetailed']; - UserWebhookBody: OneOf<[{ - note: components['schemas']['Note']; - }, { - user: components['schemas']['UserLite']; - }, { - note: components['schemas']['Note']; - reaction: { - id: string; - user: components['schemas']['UserLite']; - reaction: string; - }; - }]>; - UserWebhookNoteBody: { - note: components['schemas']['Note']; - }; - UserWebhookUserBody: { - user: components['schemas']['UserLite']; - }; - UserWebhookReactionBody: { - note: components['schemas']['Note']; - reaction: { - id: string; - user: components['schemas']['UserLite']; - reaction: string; - }; - }; UserList: { /** * Format: id diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ee3b10dfac..8c4a7c48fd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -350,6 +350,9 @@ importers: probe-image-size: specifier: 7.2.3 version: 7.2.3 + prom-client: + specifier: ^15.1.3 + version: 15.1.3 promise-limit: specifier: 2.7.0 version: 2.7.0 @@ -5252,6 +5255,9 @@ packages: resolution: {integrity: sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==} engines: {node: '>=8'} + bintrees@1.0.2: + resolution: {integrity: sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==} + blob-util@2.0.2: resolution: {integrity: sha512-T7JQa+zsXXEa6/8ZhHcQEW1UFfVM49Ts65uBkFL6fz2QmrElqmbajIDJvuA0tEhRe5eIjpV9ZF+0RfZR9voJFQ==} @@ -9329,6 +9335,10 @@ packages: resolution: {integrity: sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==} engines: {node: '>=0.4.0'} + prom-client@15.1.3: + resolution: {integrity: sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==} + engines: {node: ^16 || ^18 || >=20} + promise-limit@2.7.0: resolution: {integrity: sha512-7nJ6v5lnJsXwGprnGXga4wx6d1POjvi5Qmf1ivTRxTjH4Z/9Czja/UCMLVmB9N93GeWOU93XaFaEt6jbuoagNw==} @@ -10345,6 +10355,9 @@ packages: resolution: {integrity: sha512-+HRtZ40Vc+6YfCDWCeAsixwxJgMbPY4HHuTgzPYH3JXvqHWUlsCfy+ylXlAKhFNcuLp4xVeWeFBUhDk+7KYUvQ==} engines: {node: '>=14.16'} + tdigest@0.1.2: + resolution: {integrity: sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==} + terser@5.36.0: resolution: {integrity: sha512-IYV9eNMuFAV4THUspIRXkLakHnV6XO7FEdtKjf/mDyrnqUg9LnlOn6/RwRvM9SZjR4GUq8Nk8zj67FzVARr74w==} engines: {node: '>=10'} @@ -16086,6 +16099,8 @@ snapshots: binary-extensions@2.2.0: {} + bintrees@1.0.2: {} + blob-util@2.0.2: {} bluebird@3.7.2: {} @@ -21097,6 +21112,11 @@ snapshots: progress@2.0.3: optional: true + prom-client@15.1.3: + dependencies: + '@opentelemetry/api': 1.9.0 + tdigest: 0.1.2 + promise-limit@2.7.0: {} promise-polyfill@8.3.0: {} @@ -22280,6 +22300,10 @@ snapshots: dependencies: execa: 6.1.0 + tdigest@0.1.2: + dependencies: + bintrees: 1.0.2 + terser@5.36.0: dependencies: '@jridgewell/source-map': 0.3.6 diff --git a/yume-mods/etc/config.alloy b/yume-mods/etc/config.alloy new file mode 100644 index 0000000000..5594f09dd6 --- /dev/null +++ b/yume-mods/etc/config.alloy @@ -0,0 +1,18 @@ +prometheus.scrape "yumechinokuni" { + targets = [ + { + __address__ = "mi.yumechi.jp:443", + __scheme__ = "https", + environment = "prod", + hostname = "mi.yumechi.jp", + }, + { + __address__ = "test0.mi.yumechi.jp:443", + __scheme__ = "https", + environment = "test", + hostname = "test0.mi.yumechi.jp", + } + ] + + forward_to = [prometheus.remote_write.mihari.receiver] +} From 109d8f800894bf8e3d89b6adb688d4c8f0134434 Mon Sep 17 00:00:00 2001 From: eternal-flame-AD Date: Sun, 17 Nov 2024 06:01:25 -0600 Subject: [PATCH 3/3] don't log metrics for postgres in testing Signed-off-by: eternal-flame-AD --- package.json | 8 +- packages/backend/scripts/check_connect.js | 8 +- packages/backend/src/boot/master.ts | 13 +- packages/backend/src/core/QueueModule.ts | 20 +-- .../src/core/activitypub/ApInboxService.ts | 47 ++++--- packages/backend/src/postgres.ts | 18 +-- .../src/queue/QueueProcessorService.ts | 16 +-- packages/backend/src/queue/metrics.ts | 12 +- .../queue/processors/InboxProcessorService.ts | 117 ++++++++-------- packages/backend/src/server/ServerService.ts | 132 +++++++++--------- .../src/server/api/AuthenticateService.ts | 17 +-- .../backend/src/server/api/MetricsService.ts | 24 ++++ .../src/server/api/SigninApiService.ts | 37 ++--- packages/backend/test-server/entry.ts | 7 +- pnpm-lock.yaml | 6 +- 15 files changed, 257 insertions(+), 225 deletions(-) diff --git a/package.json b/package.json index 73b3ea7f34..d9936c275a 100644 --- a/package.json +++ b/package.json @@ -24,8 +24,8 @@ "build": "pnpm build-pre && pnpm -r build && pnpm build-assets", "build-storybook": "pnpm --filter frontend build-storybook", "build-misskey-js-with-types": "pnpm build-pre && pnpm --filter backend... --filter=!misskey-js build && pnpm --filter backend generate-api-json --no-build && ncp packages/backend/built/api.json packages/misskey-js/generator/api.json && pnpm --filter misskey-js update-autogen-code && pnpm --filter misskey-js build && pnpm --filter misskey-js api", - "start": "pnpm check:connect && cd packages/backend && node ./built/boot/entry.js", - "start:test": "cd packages/backend && cross-env NODE_ENV=test node ./built/boot/entry.js", + "start": "pnpm check:connect && cd packages/backend && cross-env RUN_MODE=web node ./built/boot/entry.js", + "start:test": "cd packages/backend && cross-env NODE_ENV=test RUN_MODE=test node ./built/boot/entry.js", "init": "pnpm migrate", "migrate": "cd packages/backend && pnpm migrate", "revert": "cd packages/backend && pnpm revert", @@ -61,14 +61,14 @@ "terser": "5.36.0", "typescript": "5.6.3", "esbuild": "0.24.0", - "glob": "11.0.0" + "glob": "11.0.0", + "cross-env": "7.0.3" }, "devDependencies": { "@misskey-dev/eslint-plugin": "2.0.3", "@types/node": "22.9.0", "@typescript-eslint/eslint-plugin": "7.17.0", "@typescript-eslint/parser": "7.17.0", - "cross-env": "7.0.3", "cypress": "13.15.2", "eslint": "9.14.0", "globals": "15.12.0", diff --git a/packages/backend/scripts/check_connect.js b/packages/backend/scripts/check_connect.js index 5d6d2fca49..fae0da3ef2 100644 --- a/packages/backend/scripts/check_connect.js +++ b/packages/backend/scripts/check_connect.js @@ -7,11 +7,6 @@ import Redis from 'ioredis'; import { loadConfig } from '../built/config.js'; import { createPostgresDataSource } from '../built/postgres.js'; -const timeout = setTimeout(() => { - console.error('Timeout while connecting to databases.'); - process.exit(1); -}, 120000); - const config = loadConfig(); async function connectToPostgres() { @@ -27,6 +22,7 @@ async function connectToRedis(redisOptions) { lazyConnect: true, reconnectOnError: false, showFriendlyErrorStack: true, + connectTimeout: 10000, }); redis.on('error', e => reject(e)); @@ -60,4 +56,4 @@ const promises = Array await Promise.allSettled(promises); -clearTimeout(timeout); +process.exit(0); diff --git a/packages/backend/src/boot/master.ts b/packages/backend/src/boot/master.ts index 551ff984b1..c3a0dc6b98 100644 --- a/packages/backend/src/boot/master.ts +++ b/packages/backend/src/boot/master.ts @@ -20,6 +20,7 @@ import type { Config } from '@/config.js'; import { showMachineInfo } from '@/misc/show-machine-info.js'; import { envOption } from '@/env.js'; import { jobQueue, server } from './common.js'; +import { metricGauge } from '@/server/api/MetricsService.js'; const _filename = fileURLToPath(import.meta.url); const _dirname = dirname(_filename); @@ -31,18 +32,24 @@ const bootLogger = logger.createSubLogger('boot', 'magenta'); const themeColor = chalk.hex('#86b300'); -const mBuildInfo = new prom.Gauge({ +const mBuildInfo = metricGauge({ name: 'misskey_build_info', help: 'Misskey build information', labelNames: ['gitCommit', 'gitDescribe', 'node_version'] }); -mBuildInfo.set({ +mBuildInfo?.set({ gitCommit: meta.gitCommit || 'unknown', gitDescribe: meta.gitDescribe || 'unknown', node_version: process.version }, 1); +const mStartupTime = metricGauge({ + name: 'misskey_startup_time', + help: 'Misskey startup time', + labelNames: ['pid'] +}); + function greet() { if (!envOption.quiet) { //#region Misskey logo @@ -112,6 +119,8 @@ export async function masterMain() { }); } + mStartupTime?.set({ pid: process.pid }, Date.now()); + if (envOption.disableClustering) { if (envOption.onlyServer) { await server(); diff --git a/packages/backend/src/core/QueueModule.ts b/packages/backend/src/core/QueueModule.ts index 3fb05cda3b..c3c9256e0e 100644 --- a/packages/backend/src/core/QueueModule.ts +++ b/packages/backend/src/core/QueueModule.ts @@ -31,16 +31,18 @@ export type UserWebhookDeliverQueue = Bull.Queue; export type SystemWebhookDeliverQueue = Bull.Queue; function withMetrics(queue: Bull.Queue): Bull.Queue { - setInterval(async () => { - mActiveJobs.set({ queue: queue.name }, await queue.getActiveCount()); - mDelayedJobs.set({ queue: queue.name }, await queue.getDelayedCount()); - mWaitingJobs.set({ queue: queue.name }, await queue.getWaitingCount()); - }, 2000); - - queue.on('waiting', () => { - mJobReceivedCounter.inc({ queue: queue.name }); - }); + if (process.env.NODE_ENV !== 'test') { + setInterval(async () => { + mActiveJobs?.set({ queue: queue.name }, await queue.getActiveCount()); + mDelayedJobs?.set({ queue: queue.name }, await queue.getDelayedCount()); + mWaitingJobs?.set({ queue: queue.name }, await queue.getWaitingCount()); + }, 2000); + queue.on('waiting', () => { + mJobReceivedCounter?.inc({ queue: queue.name }); + }); + } + return queue; } diff --git a/packages/backend/src/core/activitypub/ApInboxService.ts b/packages/backend/src/core/activitypub/ApInboxService.ts index a3bd5678d4..f3aa46292e 100644 --- a/packages/backend/src/core/activitypub/ApInboxService.ts +++ b/packages/backend/src/core/activitypub/ApInboxService.ts @@ -39,17 +39,18 @@ import { ApPersonService } from './models/ApPersonService.js'; import { ApQuestionService } from './models/ApQuestionService.js'; import type { Resolver } from './ApResolverService.js'; import type { IAccept, IAdd, IAnnounce, IBlock, ICreate, IDelete, IFlag, IFollow, ILike, IObject, IReject, IRemove, IUndo, IUpdate, IMove, IPost } from './type.js'; +import { metricCounter } from '@/server/api/MetricsService.js'; + +const mInboxReceived = metricCounter({ + name: 'misskey_ap_inbox_received_total', + help: 'Total number of activities received by AP inbox', + labelNames: ['host', 'type'], +}); @Injectable() export class ApInboxService { private logger: Logger; - private mInboxReceived = new prom.Counter({ - name: 'misskey_ap_inbox_received_total', - help: 'Total number of activities received by AP inbox', - labelNames: ['host', 'type'], - }); - constructor( @Inject(DI.config) private config: Config, @@ -138,49 +139,49 @@ export class ApInboxService { if (actor.isSuspended) return; if (isCreate(activity)) { - this.mInboxReceived.inc({ host: actor.host, type: 'create' }); + mInboxReceived?.inc({ host: actor.host, type: 'create' }); return await this.create(actor, activity); } else if (isDelete(activity)) { - this.mInboxReceived.inc({ host: actor.host, type: 'delete' }); + mInboxReceived?.inc({ host: actor.host, type: 'delete' }); return await this.delete(actor, activity); } else if (isUpdate(activity)) { - this.mInboxReceived.inc({ host: actor.host, type: 'update' }); + mInboxReceived?.inc({ host: actor.host, type: 'update' }); return await this.update(actor, activity); } else if (isFollow(activity)) { - this.mInboxReceived.inc({ host: actor.host, type: 'follow' }); + mInboxReceived?.inc({ host: actor.host, type: 'follow' }); return await this.follow(actor, activity); } else if (isAccept(activity)) { - this.mInboxReceived.inc({ host: actor.host, type: 'accept' }); + mInboxReceived?.inc({ host: actor.host, type: 'accept' }); return await this.accept(actor, activity); } else if (isReject(activity)) { - this.mInboxReceived.inc({ host: actor.host, type: 'reject' }); + mInboxReceived?.inc({ host: actor.host, type: 'reject' }); return await this.reject(actor, activity); } else if (isAdd(activity)) { - this.mInboxReceived.inc({ host: actor.host, type: 'add' }); + mInboxReceived?.inc({ host: actor.host, type: 'add' }); return await this.add(actor, activity); } else if (isRemove(activity)) { - this.mInboxReceived.inc({ host: actor.host, type: 'remove' }); + mInboxReceived?.inc({ host: actor.host, type: 'remove' }); return await this.remove(actor, activity); } else if (isAnnounce(activity)) { - this.mInboxReceived.inc({ host: actor.host, type: 'announce' }); + mInboxReceived?.inc({ host: actor.host, type: 'announce' }); return await this.announce(actor, activity); } else if (isLike(activity)) { - this.mInboxReceived.inc({ host: actor.host, type: 'like' }); + mInboxReceived?.inc({ host: actor.host, type: 'like' }); return await this.like(actor, activity); } else if (isUndo(activity)) { - this.mInboxReceived.inc({ host: actor.host, type: 'undo' }); + mInboxReceived?.inc({ host: actor.host, type: 'undo' }); return await this.undo(actor, activity); } else if (isBlock(activity)) { - this.mInboxReceived.inc({ host: actor.host, type: 'block' }); + mInboxReceived?.inc({ host: actor.host, type: 'block' }); return await this.block(actor, activity); } else if (isFlag(activity)) { - this.mInboxReceived.inc({ host: actor.host, type: 'flag' }); + mInboxReceived?.inc({ host: actor.host, type: 'flag' }); return await this.flag(actor, activity); } else if (isMove(activity)) { - this.mInboxReceived.inc({ host: actor.host, type: 'move' }); + mInboxReceived?.inc({ host: actor.host, type: 'move' }); return await this.move(actor, activity); } else { - this.mInboxReceived.inc({ host: actor.host, type: 'unknown' }); + mInboxReceived?.inc({ host: actor.host, type: 'unknown' }); return `unrecognized activity type: ${activity.type}`; } } @@ -491,9 +492,9 @@ export class ApInboxService { formerType = 'Note'; } - if (validPost.includes(formerType)) { + if (validPost?.includes(formerType)) { return await this.deleteNote(actor, uri); - } else if (validActor.includes(formerType)) { + } else if (validActor?.includes(formerType)) { return await this.deleteActor(actor, uri); } else { return `Unknown type ${formerType}`; diff --git a/packages/backend/src/postgres.ts b/packages/backend/src/postgres.ts index 6fce6a6faa..ed1fccddf0 100644 --- a/packages/backend/src/postgres.ts +++ b/packages/backend/src/postgres.ts @@ -7,7 +7,6 @@ import pg from 'pg'; import { DataSource, Logger } from 'typeorm'; import * as highlight from 'cli-highlight'; -import * as prom from 'prom-client'; import { createHash } from 'crypto'; import { entities as charts } from '@/core/chart/entities.js'; @@ -85,6 +84,7 @@ import { Config } from '@/config.js'; import MisskeyLogger from '@/logger.js'; import { bindThis } from '@/decorators.js'; import { MemoryKVCache } from './misc/cache.js'; +import { metricCounter, metricHistogram } from './server/api/MetricsService.js'; pg.types.setTypeParser(20, Number); @@ -125,19 +125,19 @@ function extractQueryTags(query: string): QueryTagCache { }; } -const mQueryCounter = new prom.Counter({ +const mQueryCounter = metricCounter({ name: 'misskey_postgres_query_total', help: 'Total queries to postgres', labelNames: ['join', 'from', 'hash'], }); -const mQueryErrorCounter = new prom.Counter({ +const mQueryErrorCounter = metricCounter({ name: 'misskey_postgres_query_error_total', help: 'Total errors in queries to postgres', labelNames: ['join', 'from', 'hash'], }); -const mSlowQueryHisto = new prom.Histogram({ +const mSlowQueryHisto = metricHistogram({ name: 'misskey_postgres_query_slow_duration_seconds', help: 'Duration of slow queries to postgres', labelNames: ['join', 'from', 'hash'], @@ -170,7 +170,7 @@ class MyCustomLogger implements Logger { @bindThis public logQuery(query: string, parameters?: any[]) { - mQueryCounter.inc(this.getQueryTags(query)); + mQueryCounter?.inc(this.getQueryTags(query)); if (this.metricOnly) { return; @@ -180,7 +180,7 @@ class MyCustomLogger implements Logger { @bindThis public logQueryError(error: string, query: string, parameters?: any[]) { - mQueryErrorCounter.inc(this.getQueryTags(query)); + mQueryErrorCounter?.inc(this.getQueryTags(query)); if (this.metricOnly) { return; @@ -190,7 +190,7 @@ class MyCustomLogger implements Logger { @bindThis public logQuerySlow(time: number, query: string, parameters?: any[]) { - mSlowQueryHisto.observe(this.getQueryTags(query), time); + mSlowQueryHisto?.observe(this.getQueryTags(query), time); if (this.metricOnly) { return; @@ -341,8 +341,8 @@ export function createPostgresDataSource(config: Config, isMain = false) { db: config.redis.db ?? 0, }, } : false, - logging: log ? 'all' : ['query'], - logger: (isMain || log) ? new MyCustomLogger(!log) : undefined, + logging: true, + logger: new MyCustomLogger(!log), maxQueryExecutionTime: 500, entities: entities, migrations: ['../../migration/*.js'], diff --git a/packages/backend/src/queue/QueueProcessorService.ts b/packages/backend/src/queue/QueueProcessorService.ts index 9a6b3d8063..b6f73c12ed 100644 --- a/packages/backend/src/queue/QueueProcessorService.ts +++ b/packages/backend/src/queue/QueueProcessorService.ts @@ -196,7 +196,7 @@ export class QueueProcessorService implements OnApplicationShutdown { }) .on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) })) .on('stalled', (jobId) => { - mStalledWorkerCounter.inc({ queue: QUEUE.SYSTEM }); + mStalledWorkerCounter?.inc({ queue: QUEUE.SYSTEM }); logger.warn(`stalled id=${jobId}`); }); } @@ -256,7 +256,7 @@ export class QueueProcessorService implements OnApplicationShutdown { }) .on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) })) .on('stalled', (jobId) => { - mStalledWorkerCounter.inc({ queue: QUEUE.DB }); + mStalledWorkerCounter?.inc({ queue: QUEUE.DB }); logger.warn(`stalled id=${jobId}`); }); } @@ -299,7 +299,7 @@ export class QueueProcessorService implements OnApplicationShutdown { }) .on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) })) .on('stalled', (jobId) => { - mStalledWorkerCounter.inc({ queue: QUEUE.DELIVER }); + mStalledWorkerCounter?.inc({ queue: QUEUE.DELIVER }); logger.warn(`stalled id=${jobId}`); }); } @@ -342,7 +342,7 @@ export class QueueProcessorService implements OnApplicationShutdown { }) .on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) })) .on('stalled', (jobId) => { - mStalledWorkerCounter.inc({ queue: QUEUE.INBOX }); + mStalledWorkerCounter?.inc({ queue: QUEUE.INBOX }); logger.warn(`stalled id=${jobId}`); }); } @@ -385,7 +385,7 @@ export class QueueProcessorService implements OnApplicationShutdown { }) .on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) })) .on('stalled', (jobId) => { - mStalledWorkerCounter.inc({ queue: QUEUE.USER_WEBHOOK_DELIVER }); + mStalledWorkerCounter?.inc({ queue: QUEUE.USER_WEBHOOK_DELIVER }); logger.warn(`stalled id=${jobId}`); }); } @@ -428,7 +428,7 @@ export class QueueProcessorService implements OnApplicationShutdown { }) .on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) })) .on('stalled', (jobId) => { - mStalledWorkerCounter.inc({ queue: QUEUE.SYSTEM_WEBHOOK_DELIVER }); + mStalledWorkerCounter?.inc({ queue: QUEUE.SYSTEM_WEBHOOK_DELIVER }); logger.warn(`stalled id=${jobId}`); }); } @@ -478,7 +478,7 @@ export class QueueProcessorService implements OnApplicationShutdown { }) .on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) })) .on('stalled', (jobId) => { - mStalledWorkerCounter.inc({ queue: QUEUE.RELATIONSHIP }); + mStalledWorkerCounter?.inc({ queue: QUEUE.RELATIONSHIP }); logger.warn(`stalled id=${jobId}`); }); } @@ -522,7 +522,7 @@ export class QueueProcessorService implements OnApplicationShutdown { }) .on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) })) .on('stalled', (jobId) => { - mStalledWorkerCounter.inc({ queue: QUEUE.OBJECT_STORAGE }); + mStalledWorkerCounter?.inc({ queue: QUEUE.OBJECT_STORAGE }); logger.warn(`stalled id=${jobId}`); }); } diff --git a/packages/backend/src/queue/metrics.ts b/packages/backend/src/queue/metrics.ts index a3e794d6a9..86c56daebc 100644 --- a/packages/backend/src/queue/metrics.ts +++ b/packages/backend/src/queue/metrics.ts @@ -1,30 +1,30 @@ -import * as prom from 'prom-client'; +import { metricCounter, metricGauge } from '@/server/api/MetricsService.js'; -export const mJobReceivedCounter = new prom.Counter({ +export const mJobReceivedCounter = metricCounter({ name: 'misskey_queue_jobs_received_total', help: 'Total number of jobs received by queue', labelNames: ['queue'], }); -export const mActiveJobs = new prom.Gauge({ +export const mActiveJobs = metricGauge({ name: 'misskey_queue_active_jobs', help: 'Number of active jobs in queue', labelNames: ['queue'], }); -export const mDelayedJobs = new prom.Gauge({ +export const mDelayedJobs = metricGauge({ name: 'misskey_queue_delayed_jobs', help: 'Number of delayed jobs in queue', labelNames: ['queue'], }); -export const mWaitingJobs = new prom.Gauge({ +export const mWaitingJobs = metricGauge({ name: 'misskey_queue_waiting_jobs', help: 'Number of waiting jobs in queue', labelNames: ['queue'], }); -export const mStalledWorkerCounter = new prom.Counter({ +export const mStalledWorkerCounter = metricCounter({ name: 'misskey_queue_stalled_workers_total', help: 'Total number of stalled workers', labelNames: ['queue'], diff --git a/packages/backend/src/queue/processors/InboxProcessorService.ts b/packages/backend/src/queue/processors/InboxProcessorService.ts index bd6da6c668..215cc66837 100644 --- a/packages/backend/src/queue/processors/InboxProcessorService.ts +++ b/packages/backend/src/queue/processors/InboxProcessorService.ts @@ -32,48 +32,49 @@ import { MiMeta } from '@/models/Meta.js'; import { DI } from '@/di-symbols.js'; import { QueueLoggerService } from '../QueueLoggerService.js'; import type { InboxJobData } from '../types.js'; +import { metricCounter, metricHistogram } from '@/server/api/MetricsService.js'; type UpdateInstanceJob = { latestRequestReceivedAt: Date, shouldUnsuspend: boolean, }; +const mIncomingApProcessingTime = metricHistogram({ + name: 'misskey_incoming_ap_processing_time', + help: 'Incoming AP processing time in seconds', + labelNames: ['incoming_host', 'incoming_type', 'success'], + buckets: [0.01, 0.1, 0.5, 1, 5, 10, 30, 60, 300, 1800], +}); + +const mIncomingApEvent = metricCounter({ + name: 'misskey_incoming_ap_event', + help: 'Incoming AP event', + labelNames: ['incoming_host', 'incoming_type'], +}); + +const mIncomingApEventAccepted = metricCounter({ + name: 'misskey_incoming_ap_event_accepted', + help: 'Incoming AP event accepted', + labelNames: ['incoming_host', 'incoming_type'], +}); + +const mIncomingApReject = metricCounter({ + name: 'misskey_incoming_ap_reject', + help: 'Incoming AP reject', + labelNames: ['incoming_host', 'incoming_type', 'reason'], +}); + +const mincomingApProcessingError = metricCounter({ + name: 'misskey_incoming_ap_processing_error', + help: 'Incoming AP processing error', + labelNames: ['incoming_host', 'incoming_type'], +}); + @Injectable() export class InboxProcessorService implements OnApplicationShutdown { private logger: Logger; private updateInstanceQueue: CollapsedQueue; - private mIncomingApProcessingTime = new prom.Histogram({ - name: 'misskey_incoming_ap_processing_time', - help: 'Incoming AP processing time in seconds', - labelNames: ['incoming_host', 'incoming_type', 'success'], - buckets: [0.01, 0.1, 0.5, 1, 5, 10, 30, 60, 300, 1800], - }); - - private mIncomingApEvent = new prom.Counter({ - name: 'misskey_incoming_ap_event', - help: 'Incoming AP event', - labelNames: ['incoming_host', 'incoming_type'], - }); - - private mIncomingApEventAccepted = new prom.Counter({ - name: 'misskey_incoming_ap_event_accepted', - help: 'Incoming AP event accepted', - labelNames: ['incoming_host', 'incoming_type'], - }); - - private mIncomingApReject = new prom.Counter({ - name: 'misskey_incoming_ap_reject', - help: 'Incoming AP reject', - labelNames: ['incoming_host', 'incoming_type', 'reason'], - }); - - private mincomingApProcessingError = new prom.Counter({ - name: 'misskey_incoming_ap_processing_error', - help: 'Incoming AP processing error', - labelNames: ['incoming_host', 'incoming_type'], - }); - constructor( @Inject(DI.meta) private meta: MiMeta, @@ -106,34 +107,34 @@ export class InboxProcessorService implements OnApplicationShutdown { const host = this.utilityService.toPuny(new URL(signature.keyId).hostname); - const incCounter = (counter: prom.Counter, addn_labels: U) => { + const incCounter = (counter: prom.Counter | null, addn_labels: U) => { if (Array.isArray(activity.type)) { for (const t of activity.type) { - counter.inc({ incoming_host: host.toString(), incoming_type: t, ...addn_labels }); + counter?.inc({ incoming_host: host.toString(), incoming_type: t, ...addn_labels }); } } else { - counter.inc({ incoming_host: host.toString(), incoming_type: activity.type ?? 'unknown', ...addn_labels }); + counter?.inc({ incoming_host: host.toString(), incoming_type: activity.type ?? 'unknown', ...addn_labels }); } }; - const observeHistogram = (histogram: prom.Histogram, addn_labels: U, value: number) => { + const observeHistogram = (histogram: prom.Histogram | null, addn_labels: U, value: number) => { if (Array.isArray(activity.type)) { for (const t of activity.type) { - histogram.observe({ incoming_host: host.toString(), incoming_type: t, ...addn_labels }, value); + histogram?.observe({ incoming_host: host.toString(), incoming_type: t, ...addn_labels }, value); } } else { - histogram.observe({ incoming_host: host.toString(), incoming_type: activity.type ?? 'unknown', ...addn_labels }, value); + histogram?.observe({ incoming_host: host.toString(), incoming_type: activity.type ?? 'unknown', ...addn_labels }, value); } }; if (!this.utilityService.isFederationAllowedHost(host)) { - incCounter(this.mIncomingApReject, { reason: 'host_not_allowed' }); + incCounter(mIncomingApReject, { reason: 'host_not_allowed' }); return `Blocked request: ${host}`; } const keyIdLower = signature.keyId.toLowerCase(); if (keyIdLower.startsWith('acct:')) { - incCounter(this.mIncomingApReject, { reason: 'keyid_acct' }); + incCounter(mIncomingApReject, { reason: 'keyid_acct' }); return `Old keyId is no longer supported. ${keyIdLower}`; } @@ -151,7 +152,7 @@ export class InboxProcessorService implements OnApplicationShutdown { // 対象が4xxならスキップ if (err instanceof StatusError) { if (!err.isRetryable) { - incCounter(this.mIncomingApReject, { reason: 'actor_key_unresolvable' }); + incCounter(mIncomingApReject, { reason: 'actor_key_unresolvable' }); throw new Bull.UnrecoverableError(`skip: Ignored deleted actors on both ends ${activity.actor} - ${err.statusCode}`); } throw new Error(`Error in actor ${activity.actor} - ${err.statusCode}`); @@ -161,13 +162,13 @@ export class InboxProcessorService implements OnApplicationShutdown { // それでもわからなければ終了 if (authUser == null) { - incCounter(this.mIncomingApReject, { reason: 'actor_unresolvable' }); + incCounter(mIncomingApReject, { reason: 'actor_unresolvable' }); throw new Bull.UnrecoverableError('skip: failed to resolve user'); } // publicKey がなくても終了 if (authUser.key == null) { - incCounter(this.mIncomingApReject, { reason: 'publickey_unresolvable' }); + incCounter(mIncomingApReject, { reason: 'publickey_unresolvable' }); throw new Bull.UnrecoverableError('skip: failed to resolve user publicKey'); } @@ -180,7 +181,7 @@ export class InboxProcessorService implements OnApplicationShutdown { const ldSignature = activity.signature; if (ldSignature) { if (ldSignature.type !== 'RsaSignature2017') { - incCounter(this.mIncomingApReject, { reason: 'ld_signature_unsupported' }); + incCounter(mIncomingApReject, { reason: 'ld_signature_unsupported' }); throw new Bull.UnrecoverableError(`skip: unsupported LD-signature type ${ldSignature.type}`); } @@ -194,12 +195,12 @@ export class InboxProcessorService implements OnApplicationShutdown { // keyIdからLD-Signatureのユーザーを取得 authUser = await this.apDbResolverService.getAuthUserFromKeyId(ldSignature.creator); if (authUser == null) { - incCounter(this.mIncomingApReject, { reason: 'ld_signature_user_unresolvable' }); + incCounter(mIncomingApReject, { reason: 'ld_signature_user_unresolvable' }); throw new Bull.UnrecoverableError('skip: LD-Signatureのユーザーが取得できませんでした'); } if (authUser.key == null) { - incCounter(this.mIncomingApReject, { reason: 'ld_signature_publickey_unavailable' }); + incCounter(mIncomingApReject, { reason: 'ld_signature_publickey_unavailable' }); throw new Bull.UnrecoverableError('skip: LD-SignatureのユーザーはpublicKeyを持っていませんでした'); } @@ -208,7 +209,7 @@ export class InboxProcessorService implements OnApplicationShutdown { // LD-Signature検証 const verified = await jsonLd.verifyRsaSignature2017(activity, authUser.key.keyPem).catch(() => false); if (!verified) { - incCounter(this.mIncomingApReject, { reason: 'ld_signature_verification_failed' }); + incCounter(mIncomingApReject, { reason: 'ld_signature_verification_failed' }); throw new Bull.UnrecoverableError('skip: LD-Signatureの検証に失敗しました'); } @@ -231,17 +232,17 @@ export class InboxProcessorService implements OnApplicationShutdown { // もう一度actorチェック if (authUser.user.uri !== activity.actor) { - incCounter(this.mIncomingApReject, { reason: 'ld_signature_actor_mismatch' }); + incCounter(mIncomingApReject, { reason: 'ld_signature_actor_mismatch' }); throw new Bull.UnrecoverableError(`skip: LD-Signature user(${authUser.user.uri}) !== activity.actor(${activity.actor})`); } const ldHost = this.utilityService.extractDbHost(authUser.user.uri); if (!this.utilityService.isFederationAllowedHost(ldHost)) { - incCounter(this.mIncomingApReject, { reason: 'fed_host_not_allowed' }); + incCounter(mIncomingApReject, { reason: 'fed_host_not_allowed' }); throw new Bull.UnrecoverableError(`Blocked request: ${ldHost}`); } } else { - incCounter(this.mIncomingApReject, { reason: 'ld_signature_unavailable' }); + incCounter(mIncomingApReject, { reason: 'ld_signature_unavailable' }); throw new Bull.UnrecoverableError(`skip: http-signature verification failed and no LD-Signature. keyId=${signature.keyId}`); } } @@ -251,7 +252,7 @@ export class InboxProcessorService implements OnApplicationShutdown { const signerHost = this.utilityService.extractDbHost(authUser.user.uri!); const activityIdHost = this.utilityService.extractDbHost(activity.id); if (signerHost !== activityIdHost) { - incCounter(this.mIncomingApReject, 'host_signature_mismatch'); + incCounter(mIncomingApReject, 'host_signature_mismatch'); throw new Bull.UnrecoverableError(`skip: signerHost(${signerHost}) !== activity.id host(${activityIdHost}`); } } @@ -279,7 +280,7 @@ export class InboxProcessorService implements OnApplicationShutdown { this.fetchInstanceMetadataService.fetchInstanceMetadata(i); }); - incCounter(this.mIncomingApEvent, {}); + incCounter(mIncomingApEvent, {}); // アクティビティを処理 const begin = +new Date(); @@ -292,25 +293,25 @@ export class InboxProcessorService implements OnApplicationShutdown { } catch (e) { if (e instanceof IdentifiableError) { if (e.id === '689ee33f-f97c-479a-ac49-1b9f8140af99') { - incCounter(this.mIncomingApReject, { reason: 'blocked_notes_with_prohibited_words' }); + incCounter(mIncomingApReject, { reason: 'blocked_notes_with_prohibited_words' }); return 'blocked notes with prohibited words'; } if (e.id === '85ab9bd7-3a41-4530-959d-f07073900109') { - incCounter(this.mIncomingApReject, { reason: 'actor_suspended' }); + incCounter(mIncomingApReject, { reason: 'actor_suspended' }); return 'actor has been suspended'; } if (e.id === 'd450b8a9-48e4-4dab-ae36-f4db763fda7c') { // invalid Note - incCounter(this.mIncomingApReject, { reason: 'invalid_note' }); + incCounter(mIncomingApReject, { reason: 'invalid_note' }); return e.message; } } const end = +new Date(); - observeHistogram(this.mIncomingApProcessingTime, { success: 'false' }, (end - begin) / 1000); - incCounter(this.mincomingApProcessingError, { reason: 'unknown' }); + observeHistogram(mIncomingApProcessingTime, { success: 'false' }, (end - begin) / 1000); + incCounter(mincomingApProcessingError, { reason: 'unknown' }); throw e; } - observeHistogram(this.mIncomingApProcessingTime, { success: 'true' }, (+new Date() - begin) / 1000); - incCounter(this.mIncomingApEventAccepted, {}); + observeHistogram(mIncomingApProcessingTime, { success: 'true' }, (+new Date() - begin) / 1000); + incCounter(mIncomingApEventAccepted, {}); return 'ok'; } diff --git a/packages/backend/src/server/ServerService.ts b/packages/backend/src/server/ServerService.ts index bbb0503c1e..dc48929112 100644 --- a/packages/backend/src/server/ServerService.ts +++ b/packages/backend/src/server/ServerService.ts @@ -35,7 +35,7 @@ import { makeHstsHook } from './hsts.js'; import { generateCSP } from './csp.js'; import * as prom from 'prom-client'; import { sanitizeRequestURI } from '@/misc/log-sanitization.js'; -import { MetricsService } from './api/MetricsService.js'; +import { metricCounter, metricGauge, metricHistogram, MetricsService } from './api/MetricsService.js'; const _dirname = fileURLToPath(new URL('.', import.meta.url)); @@ -55,66 +55,66 @@ function categorizeRequestPath(path: string): 'api' | 'health' | 'vite' | 'other return 'other'; } +const mRequestTime = metricHistogram({ + name: 'misskey_http_request_duration_seconds', + help: 'Duration of handling HTTP requests in seconds', + labelNames: ['host', 'cate', 'method', 'path'], + buckets: [0.001, 0.1, 0.5, 1, 2, 5], +}); + +const mRequestsReceived = metricCounter({ + name: 'misskey_http_requests_received_total', + help: 'Total number of HTTP requests received', + labelNames: [], +}); + +const mNotFoundServed = metricCounter({ + name: 'misskey_http_not_found_served_total', + help: 'Total number of HTTP 404 responses served', + labelNames: ['method', 'cate'], +}); + +const mMethodNotAllowedServed = metricCounter({ + name: 'misskey_http_method_not_allowed_served_total', + help: 'Total number of HTTP 405 responses served', + labelNames: ['method', 'cate'], +}); + +const mTooManyRequestsServed = metricCounter({ + name: 'misskey_http_too_many_requests_served_total', + help: 'Total number of HTTP 429 responses served', + labelNames: ['method', 'cate'], +}); + +const mAggregateRequestsServed = metricCounter({ + name: 'misskey_http_requests_served_total', + help: 'Total number of HTTP requests served including invalid requests', + labelNames: ['host', 'cate', 'status'], +}); + +const mRequestsServedByPath = metricCounter({ + name: 'misskey_http_requests_served_by_path', + help: 'Total number of HTTP requests served', + labelNames: ['host', 'cate', 'method', 'path', 'status'], +}); + +const mFatalErrorCount = metricCounter({ + name: 'misskey_fatal_http_errors_total', + help: 'Total number of HTTP errors that propagate to the top level', + labelNames: ['host', 'cate', 'method', 'path'], +}); + +const mLastSuccessfulRequest = metricGauge({ + name: 'misskey_http_last_successful_request_timestamp_seconds', + help: 'Unix Timestamp of the last successful HTTP request', + labelNames: [], +}); + @Injectable() export class ServerService implements OnApplicationShutdown { private logger: Logger; #fastify: FastifyInstance; - private mRequestTime = new prom.Histogram({ - name: 'misskey_http_request_duration_seconds', - help: 'Duration of handling HTTP requests in seconds', - labelNames: ['host', 'cate', 'method', 'path'], - buckets: [0.001, 0.1, 0.5, 1, 2, 5], - }); - - private mRequestsReceived = new prom.Counter({ - name: 'misskey_http_requests_received_total', - help: 'Total number of HTTP requests received', - labelNames: [], - }); - - private mNotFoundServed = new prom.Counter({ - name: 'misskey_http_not_found_served_total', - help: 'Total number of HTTP 404 responses served', - labelNames: ['method', 'cate'], - }); - - private mMethodNotAllowedServed = new prom.Counter({ - name: 'misskey_http_method_not_allowed_served_total', - help: 'Total number of HTTP 405 responses served', - labelNames: ['method', 'cate'], - }); - - private mTooManyRequestsServed = new prom.Counter({ - name: 'misskey_http_too_many_requests_served_total', - help: 'Total number of HTTP 429 responses served', - labelNames: ['method', 'cate'], - }); - - private mAggregateRequestsServed = new prom.Counter({ - name: 'misskey_http_requests_served_total', - help: 'Total number of HTTP requests served including invalid requests', - labelNames: ['host', 'cate', 'status'], - }); - - private mRequestsServedByPath = new prom.Counter({ - name: 'misskey_http_requests_served_by_path', - help: 'Total number of HTTP requests served', - labelNames: ['host', 'cate', 'method', 'path', 'status'], - }); - - private mFatalErrorCount = new prom.Counter({ - name: 'misskey_fatal_http_errors_total', - help: 'Total number of HTTP errors that propagate to the top level', - labelNames: ['host', 'cate', 'method', 'path'], - }); - - private mLastSuccessfulRequest = new prom.Gauge({ - name: 'misskey_http_last_successful_request_timestamp_seconds', - help: 'Unix Timestamp of the last successful HTTP request', - labelNames: [], - }); - constructor( @Inject(DI.config) private config: Config, @@ -160,14 +160,14 @@ export class ServerService implements OnApplicationShutdown { if (this.config.prometheusMetrics?.enable) { fastify.addHook('onRequest', (_request, reply, done) => { reply.header('x-request-received', (+new Date()).toString()); - this.mRequestsReceived.inc(); + mRequestsReceived?.inc(); done(); }); fastify.addHook('onError', (request, _reply, error, done) => { const url = new URL(request.url, this.config.url); const logPath = sanitizeRequestURI(url.pathname); - this.mFatalErrorCount.inc({ + mFatalErrorCount?.inc({ host: request.hostname, method: request.method, path: logPath, @@ -182,14 +182,14 @@ export class ServerService implements OnApplicationShutdown { const cate = categorizeRequestPath(logPath); const received = reply.getHeader('x-request-received') as string; - this.mAggregateRequestsServed.inc({ + mAggregateRequestsServed?.inc({ host: request.hostname, cate, status: reply.statusCode, }); if (reply.statusCode === 429) { - this.mTooManyRequestsServed.inc({ + mTooManyRequestsServed?.inc({ method: request.method, cate, }); @@ -199,14 +199,14 @@ export class ServerService implements OnApplicationShutdown { } if (reply.statusCode === 404) { - this.mNotFoundServed.inc({ + mNotFoundServed?.inc({ method: request.method, cate, }); if (received) { const duration = (+new Date()) - parseInt(received); - this.mRequestTime.observe({ + mRequestTime?.observe({ host: request.hostname, method: request.method, cate, @@ -218,7 +218,7 @@ export class ServerService implements OnApplicationShutdown { } if (reply.statusCode === 405) { - this.mMethodNotAllowedServed.inc({ + mMethodNotAllowedServed?.inc({ method: request.method, cate, }); @@ -229,7 +229,7 @@ export class ServerService implements OnApplicationShutdown { if (received) { const duration = (+new Date()) - parseInt(received); - this.mRequestTime.observe({ + mRequestTime?.observe({ host: request.hostname, method: request.method, cate, @@ -243,10 +243,10 @@ export class ServerService implements OnApplicationShutdown { } if (reply.statusCode <= 299) { - this.mLastSuccessfulRequest.set(+new Date() / 1000); + mLastSuccessfulRequest?.set(+new Date() / 1000); } - this.mRequestsServedByPath.inc({ + mRequestsServedByPath?.inc({ host: request.hostname, method: request.method, path: logPath, diff --git a/packages/backend/src/server/api/AuthenticateService.ts b/packages/backend/src/server/api/AuthenticateService.ts index 13a51e3797..dfb5777c13 100644 --- a/packages/backend/src/server/api/AuthenticateService.ts +++ b/packages/backend/src/server/api/AuthenticateService.ts @@ -14,6 +14,7 @@ import type { MiApp } from '@/models/App.js'; import { CacheService } from '@/core/CacheService.js'; import isNativeToken from '@/misc/is-native-token.js'; import { bindThis } from '@/decorators.js'; +import { metricCounter } from './MetricsService.js'; export class AuthenticationError extends Error { constructor(message: string) { @@ -22,16 +23,16 @@ export class AuthenticationError extends Error { } } +const mAuthenticationFailureCounter = metricCounter({ + name: 'misskey_authentication_failure_total', + help: 'Total number of authentication failures', + labelNames: ['cred_ty'], +}); + @Injectable() export class AuthenticateService implements OnApplicationShutdown { private appCache: MemoryKVCache; - private mAuthenticationFailureCounter = new prom.Counter({ - name: 'misskey_authentication_failure_total', - help: 'Total number of authentication failures', - labelNames: ['cred_ty'], - }); - constructor( @Inject(DI.usersRepository) private usersRepository: UsersRepository, @@ -58,7 +59,7 @@ export class AuthenticateService implements OnApplicationShutdown { () => this.usersRepository.findOneBy({ token }) as Promise); if (user == null) { - this.mAuthenticationFailureCounter.inc({ cred_ty: 'native' }); + mAuthenticationFailureCounter?.inc({ cred_ty: 'native' }); throw new AuthenticationError('user not found'); } @@ -73,7 +74,7 @@ export class AuthenticateService implements OnApplicationShutdown { }); if (accessToken == null) { - this.mAuthenticationFailureCounter.inc({ cred_ty: 'access_token' }); + mAuthenticationFailureCounter?.inc({ cred_ty: 'access_token' }); throw new AuthenticationError('invalid signature'); } diff --git a/packages/backend/src/server/api/MetricsService.ts b/packages/backend/src/server/api/MetricsService.ts index 034098960d..2837fdbb19 100644 --- a/packages/backend/src/server/api/MetricsService.ts +++ b/packages/backend/src/server/api/MetricsService.ts @@ -5,6 +5,30 @@ import type { Config } from '@/config.js'; import { bindThis } from "@/decorators.js"; import type { FastifyInstance, FastifyPluginOptions } from "fastify"; +export function metricGauge(conf: prom.GaugeConfiguration) : prom.Gauge | null { + if (!process.env.RUN_MODE) { + return null; + } + + return new prom.Gauge(conf); +} + +export function metricCounter(conf: prom.CounterConfiguration) : prom.Counter | null { + if (!process.env.RUN_MODE) { + return null; + } + + return new prom.Counter(conf); +} + +export function metricHistogram(conf: prom.HistogramConfiguration) : prom.Histogram | null { + if (!process.env.RUN_MODE) { + return null; + } + + return new prom.Histogram(conf); +} + /* * SPDX-FileCopyrightText: syuilo and misskey-project and yumechi * SPDX-License-Identifier: AGPL-3.0-only diff --git a/packages/backend/src/server/api/SigninApiService.ts b/packages/backend/src/server/api/SigninApiService.ts index 80cfdf5436..0327f17ea5 100644 --- a/packages/backend/src/server/api/SigninApiService.ts +++ b/packages/backend/src/server/api/SigninApiService.ts @@ -29,15 +29,16 @@ import { RateLimiterService } from './RateLimiterService.js'; import { SigninService } from './SigninService.js'; import type { AuthenticationResponseJSON } from '@simplewebauthn/types'; import type { FastifyReply, FastifyRequest } from 'fastify'; +import { metricCounter } from './MetricsService.js'; + +const mSigninFailureCounter = metricCounter({ + name: 'misskey_misskey_signin_failure', + help: 'The number of failed sign-ins', + labelNames: ['reason'], +}); @Injectable() export class SigninApiService { - private mSigninFailureCounter = new prom.Counter({ - name: 'misskey_misskey_signin_failure', - help: 'The number of failed sign-ins', - labelNames: ['reason'], - }); - constructor( @Inject(DI.config) private config: Config, @@ -100,7 +101,7 @@ export class SigninApiService { // not more than 1 attempt per second and not more than 10 attempts per hour await this.rateLimiterService.limit({ key: 'signin', duration: 60 * 60 * 1000, max: 10, minInterval: 1000 }, getIpHash(request.ip)); } catch (err) { - this.mSigninFailureCounter.inc({ reason: 'rate_limit' }); + mSigninFailureCounter?.inc({ reason: 'rate_limit' }); reply.code(429); return { error: { @@ -112,13 +113,13 @@ export class SigninApiService { } if (typeof username !== 'string') { - this.mSigninFailureCounter.inc({ reason: 'bad_form' }); + mSigninFailureCounter?.inc({ reason: 'bad_form' }); reply.code(400); return; } if (token != null && typeof token !== 'string') { - this.mSigninFailureCounter.inc({ reason: 'bad_form' }); + mSigninFailureCounter?.inc({ reason: 'bad_form' }); reply.code(400); return; } @@ -130,14 +131,14 @@ export class SigninApiService { }) as MiLocalUser; if (user == null) { - this.mSigninFailureCounter.inc({ reason: 'user_not_found' }); + mSigninFailureCounter?.inc({ reason: 'user_not_found' }); return error(404, { id: '6cc579cc-885d-43d8-95c2-b8c7fc963280', }); } if (user.isSuspended) { - this.mSigninFailureCounter.inc({ reason: 'user_suspended' }); + mSigninFailureCounter?.inc({ reason: 'user_suspended' }); return error(403, { id: 'e03a5f46-d309-4865-9b69-56282d94e1eb', }); @@ -162,7 +163,7 @@ export class SigninApiService { } if (typeof password !== 'string') { - this.mSigninFailureCounter.inc({ reason: 'bad_form' }); + mSigninFailureCounter?.inc({ reason: 'bad_form' }); reply.code(400); return; } @@ -180,7 +181,7 @@ export class SigninApiService { success: false, }); - this.mSigninFailureCounter.inc({ reason: failure?.id ?? `unknown_error_${status ?? 500}` }); + mSigninFailureCounter?.inc({ reason: failure?.id ?? `unknown_error_${status ?? 500}` }); return error(status ?? 500, failure ?? { id: '4e30e80c-e338-45a0-8c8f-44455efa3b76' }); }; @@ -188,35 +189,35 @@ export class SigninApiService { if (process.env.NODE_ENV !== 'test') { if (this.meta.enableHcaptcha && this.meta.hcaptchaSecretKey) { await this.captchaService.verifyHcaptcha(this.meta.hcaptchaSecretKey, body['hcaptcha-response']).catch(err => { - this.mSigninFailureCounter.inc({ reason: 'captcha_verification_failed_hcaptcha' }); + mSigninFailureCounter?.inc({ reason: 'captcha_verification_failed_hcaptcha' }); throw new FastifyReplyError(400, err); }); } if (this.meta.enableMcaptcha && this.meta.mcaptchaSecretKey && this.meta.mcaptchaSitekey && this.meta.mcaptchaInstanceUrl) { await this.captchaService.verifyMcaptcha(this.meta.mcaptchaSecretKey, this.meta.mcaptchaSitekey, this.meta.mcaptchaInstanceUrl, body['m-captcha-response']).catch(err => { - this.mSigninFailureCounter.inc({ reason: 'captcha_verification_failed_mcaptcha' }); + mSigninFailureCounter?.inc({ reason: 'captcha_verification_failed_mcaptcha' }); throw new FastifyReplyError(400, err); }); } if (this.meta.enableRecaptcha && this.meta.recaptchaSecretKey) { await this.captchaService.verifyRecaptcha(this.meta.recaptchaSecretKey, body['g-recaptcha-response']).catch(err => { - this.mSigninFailureCounter.inc({ reason: 'captcha_verification_failed_recaptcha' }); + mSigninFailureCounter?.inc({ reason: 'captcha_verification_failed_recaptcha' }); throw new FastifyReplyError(400, err); }); } if (this.meta.enableTurnstile && this.meta.turnstileSecretKey) { await this.captchaService.verifyTurnstile(this.meta.turnstileSecretKey, body['turnstile-response']).catch(err => { - this.mSigninFailureCounter.inc({ reason: 'captcha_verification_failed_turnstile' }); + mSigninFailureCounter?.inc({ reason: 'captcha_verification_failed_turnstile' }); throw new FastifyReplyError(400, err); }); } if (this.meta.enableTestcaptcha) { await this.captchaService.verifyTestcaptcha(body['testcaptcha-response']).catch(err => { - this.mSigninFailureCounter.inc({ reason: 'captcha_verification_failed_testcaptcha' }); + mSigninFailureCounter?.inc({ reason: 'captcha_verification_failed_testcaptcha' }); throw new FastifyReplyError(400, err); }); } diff --git a/packages/backend/test-server/entry.ts b/packages/backend/test-server/entry.ts index 04bf62d209..a6266e3440 100644 --- a/packages/backend/test-server/entry.ts +++ b/packages/backend/test-server/entry.ts @@ -5,7 +5,6 @@ import { NestFactory } from '@nestjs/core'; import { MainModule } from '@/MainModule.js'; import { ServerService } from '@/server/ServerService.js'; import { loadConfig } from '@/config.js'; -import { NestLogger } from '@/NestLogger.js'; import { INestApplicationContext } from '@nestjs/common'; const config = loadConfig(); @@ -22,10 +21,8 @@ let serverService: ServerService; async function launch() { await killTestServer(); - console.log('starting application...'); - app = await NestFactory.createApplicationContext(MainModule, { - logger: new NestLogger(), + logger: ["debug", "log", "error", "warn", "verbose"], }); serverService = app.get(ServerService); await serverService.launch(); @@ -84,7 +81,7 @@ async function startControllerEndpoints(port = config.port + 1000) { console.log('starting application...'); app = await NestFactory.createApplicationContext(MainModule, { - logger: new NestLogger(), + logger: ["debug", "log", "error", "warn", "verbose"], }); serverService = app.get(ServerService); await serverService.launch(); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8c4a7c48fd..891b83d207 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12,6 +12,9 @@ importers: .: dependencies: + cross-env: + specifier: 7.0.3 + version: 7.0.3 cssnano: specifier: 6.1.2 version: 6.1.2(postcss@8.4.49) @@ -62,9 +65,6 @@ importers: '@typescript-eslint/parser': specifier: 7.17.0 version: 7.17.0(eslint@9.14.0)(typescript@5.6.3) - cross-env: - specifier: 7.0.3 - version: 7.0.3 cypress: specifier: 13.15.2 version: 13.15.2