Merge pull request 'prometheus' (#20) from develop into master

Reviewed-on: #20
2024-11-17 12:40:23 -06:00 · 2024-11-17 12:40:23 -06:00 · 577a7e5e96
commit 577a7e5e96
parent a95e3716ff 109d8f8008
32 changed files with 757 additions and 89 deletions
--- a/.config/cypress-devcontainer.yml
+++ b/.config/cypress-devcontainer.yml
@ -153,6 +153,13 @@ redis:

 id: 'aidx'

+#   ┌──────────┐
+#───┘ Metrics  └──────────────────────────────────────────
+
+#prometheusMetrics:
+# enable: false
+# scrapeToken: '' # Set non-empty to require a bearer token for scraping
+
 #   ┌────────────────┐
 #───┘ Error tracking └──────────────────────────────────────────

--- a/.config/docker_example.yml
+++ b/.config/docker_example.yml
@ -147,6 +147,13 @@ redis:

 id: 'aidx'

+#   ┌──────────┐
+#───┘ Metrics  └──────────────────────────────────────────
+
+#prometheusMetrics:
+# enable: false
+# scrapeToken: '' # Set non-empty to require a bearer token for scraping
+
 #   ┌────────────────┐
 #───┘ Error tracking └──────────────────────────────────────────

--- a/.config/example.yml
+++ b/.config/example.yml
@ -229,6 +229,13 @@ redis:

 id: 'aidx'

+#   ┌──────────┐
+#───┘ Metrics  └──────────────────────────────────────────
+
+#prometheusMetrics:
+# enable: false
+# scrapeToken: '' # Set non-empty to require a bearer token for scraping
+
 #   ┌────────────────┐
 #───┘ Error tracking └──────────────────────────────────────────

--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -9,6 +9,10 @@

 PgroongaのCWサーチ (github.com/paricafe/misskey#d30db97b59d264450901c1dd86808dcb43875ea9)

+### 2024.11.0-yumechinokuni.4p2
+
+- fix(backend): アナウンスメントを作成ときにWebUIフォームの画像URLを後悔できないのを修正 (/admin/announcement/create)
+
 ## 2024.11.0-yumechinokuni.3

 - Security: CSPの設定を強化
--- a/package.json
+++ b/package.json
@ -24,8 +24,8 @@
 		"build": "pnpm build-pre && pnpm -r build && pnpm build-assets",
 		"build-storybook": "pnpm --filter frontend build-storybook",
 		"build-misskey-js-with-types": "pnpm build-pre && pnpm --filter backend... --filter=!misskey-js build && pnpm --filter backend generate-api-json --no-build && ncp packages/backend/built/api.json packages/misskey-js/generator/api.json && pnpm --filter misskey-js update-autogen-code && pnpm --filter misskey-js build && pnpm --filter misskey-js api",
-		"start": "pnpm check:connect && cd packages/backend && node ./built/boot/entry.js",
-		"start:test": "cd packages/backend && cross-env NODE_ENV=test node ./built/boot/entry.js",
+		"start": "pnpm check:connect && cd packages/backend && cross-env RUN_MODE=web node ./built/boot/entry.js",
+		"start:test": "cd packages/backend && cross-env NODE_ENV=test RUN_MODE=test node ./built/boot/entry.js",
 		"init": "pnpm migrate",
 		"migrate": "cd packages/backend && pnpm migrate",
 		"revert": "cd packages/backend && pnpm revert",
@ -61,14 +61,14 @@
 		"terser": "5.36.0",
 		"typescript": "5.6.3",
 		"esbuild": "0.24.0",
-		"glob": "11.0.0"
+		"glob": "11.0.0",
+		"cross-env": "7.0.3"
 	},
 	"devDependencies": {
 		"@misskey-dev/eslint-plugin": "2.0.3",
 		"@types/node": "22.9.0",
 		"@typescript-eslint/eslint-plugin": "7.17.0",
 		"@typescript-eslint/parser": "7.17.0",
-		"cross-env": "7.0.3",
 		"cypress": "13.15.2",
 		"eslint": "9.14.0",
 		"globals": "15.12.0",
--- a/packages/backend/package.json
+++ b/packages/backend/package.json
@ -134,8 +134,8 @@
 		"json5": "2.2.3",
 		"jsonld": "8.3.2",
 		"jsrsasign": "11.1.0",
-		"meilisearch": "0.45.0",
 		"juice": "11.0.0",
+		"meilisearch": "0.45.0",
 		"mfm-js": "0.24.0",
 		"microformats-parser": "2.0.2",
 		"mime-types": "2.1.35",
@ -156,6 +156,7 @@
 		"pg": "8.13.1",
 		"pkce-challenge": "4.1.0",
 		"probe-image-size": "7.2.3",
+		"prom-client": "^15.1.3",
 		"promise-limit": "2.7.0",
 		"pug": "3.0.3",
 		"punycode": "2.3.1",
--- a/packages/backend/scripts/check_connect.js
+++ b/packages/backend/scripts/check_connect.js
@ -22,6 +22,7 @@ async function connectToRedis(redisOptions) {
 			lazyConnect: true,
 			reconnectOnError: false,
 			showFriendlyErrorStack: true,
+			connectTimeout: 10000,
 		});
 		redis.on('error', e => reject(e));

@ -50,7 +51,9 @@ const promises = Array
 	]))
 	.map(connectToRedis)
 	.concat([
-		connectToPostgres()
+		connectToPostgres().then(() => { console.log('Connected to PostgreSQL.'); }),
 	]);

 await Promise.allSettled(promises);
+
+process.exit(0);
--- a/packages/backend/src/GlobalModule.ts
+++ b/packages/backend/src/GlobalModule.ts
@ -24,7 +24,7 @@ const $config: Provider = {
 const $db: Provider = {
 	provide: DI.db,
 	useFactory: async (config) => {
-		const db = createPostgresDataSource(config);
+		const db = createPostgresDataSource(config, true);
 		return await db.initialize();
 	},
 	inject: [DI.config],
--- a/packages/backend/src/boot/master.ts
+++ b/packages/backend/src/boot/master.ts
@ -7,9 +7,11 @@ import * as fs from 'node:fs';
 import { fileURLToPath } from 'node:url';
 import { dirname } from 'node:path';
 import * as os from 'node:os';
+import * as prom from 'prom-client';
 import cluster from 'node:cluster';
 import chalk from 'chalk';
 import chalkTemplate from 'chalk-template';
+import { collectDefaultMetrics } from 'prom-client';
 import * as Sentry from '@sentry/node';
 import { nodeProfilingIntegration } from '@sentry/profiling-node';
 import Logger from '@/logger.js';
@ -18,6 +20,7 @@ import type { Config } from '@/config.js';
 import { showMachineInfo } from '@/misc/show-machine-info.js';
 import { envOption } from '@/env.js';
 import { jobQueue, server } from './common.js';
+import { metricGauge } from '@/server/api/MetricsService.js';

 const _filename = fileURLToPath(import.meta.url);
 const _dirname = dirname(_filename);
@ -29,6 +32,24 @@ const bootLogger = logger.createSubLogger('boot', 'magenta');

 const themeColor = chalk.hex('#86b300');

+const mBuildInfo = metricGauge({
+	name: 'misskey_build_info',
+	help: 'Misskey build information',
+	labelNames: ['gitCommit', 'gitDescribe', 'node_version']
+});
+
+mBuildInfo?.set({
+	gitCommit: meta.gitCommit || 'unknown',
+	gitDescribe: meta.gitDescribe || 'unknown',
+	node_version: process.version
+}, 1);
+
+const mStartupTime = metricGauge({
+	name: 'misskey_startup_time',
+	help: 'Misskey startup time',
+	labelNames: ['pid']
+});
+
 function greet() {
 	if (!envOption.quiet) {
 		//#region Misskey logo
@ -64,6 +85,13 @@ export async function masterMain() {
 		await showMachineInfo(bootLogger);
 		showNodejsVersion();
 		config = loadConfigBoot();
+
+		collectDefaultMetrics({
+			labels: {
+				cluster_type: 'master',
+			}
+		});
+
 		//await connectDb();
 		if (config.pidFile) fs.writeFileSync(config.pidFile, process.pid.toString());
 	} catch (e) {
@ -91,6 +119,8 @@ export async function masterMain() {
 		});
 	}

+	mStartupTime?.set({ pid: process.pid }, Date.now());
+
 	if (envOption.disableClustering) {
 		if (envOption.onlyServer) {
 			await server();
--- a/packages/backend/src/boot/worker.ts
+++ b/packages/backend/src/boot/worker.ts
@ -4,6 +4,7 @@
 */

 import cluster from 'node:cluster';
+import { collectDefaultMetrics } from 'prom-client';
 import * as Sentry from '@sentry/node';
 import { nodeProfilingIntegration } from '@sentry/profiling-node';
 import { envOption } from '@/env.js';
@ -16,6 +17,12 @@ import { jobQueue, server } from './common.js';
 export async function workerMain() {
 	const config = loadConfig();

+	collectDefaultMetrics({
+		labels: {
+			cluster_type: 'worker',
+		}
+	});
+
 	if (config.sentryForBackend) {
 		Sentry.init({
 			integrations: [
--- a/packages/backend/src/config.ts
+++ b/packages/backend/src/config.ts
@ -72,6 +72,9 @@ type Source = {
 		index: string;
 		scope?: 'local' | 'global' | string[];
 	};
+	
+	prometheusMetrics?: { enable: boolean, scrapeToken?: string };
+
 	sentryForBackend?: { options: Partial<Sentry.NodeOptions>; enableNodeProfiling: boolean; };
 	sentryForFrontend?: { options: Partial<Sentry.NodeOptions> };

@ -199,8 +202,12 @@ export type Config = {
 	redisForJobQueue: RedisOptions & RedisOptionsSource;
 	redisForTimelines: RedisOptions & RedisOptionsSource;
 	redisForReactions: RedisOptions & RedisOptionsSource;
+
+	prometheusMetrics : { enable: boolean, scrapeToken?: string } | undefined;
+
 	sentryForBackend: { options: Partial<Sentry.NodeOptions>; enableNodeProfiling: boolean; } | undefined;
 	sentryForFrontend: { options: Partial<Sentry.NodeOptions> } | undefined;
+	
 	perChannelMaxNoteCacheCount: number;
 	perUserNotificationsMaxCount: number;
 	deactivateAntennaThreshold: number;
@ -295,6 +302,7 @@ export function loadConfig(): Config {
 		redisForJobQueue: config.redisForJobQueue ? convertRedisOptions(config.redisForJobQueue, host) : redis,
 		redisForTimelines: config.redisForTimelines ? convertRedisOptions(config.redisForTimelines, host) : redis,
 		redisForReactions: config.redisForReactions ? convertRedisOptions(config.redisForReactions, host) : redis,
+		prometheusMetrics: config.prometheusMetrics,
 		sentryForBackend: config.sentryForBackend,
 		sentryForFrontend: config.sentryForFrontend,
 		id: config.id,
--- a/packages/backend/src/core/AnnouncementService.ts
+++ b/packages/backend/src/core/AnnouncementService.ts
@ -72,7 +72,7 @@ export class AnnouncementService {
 			updatedAt: null,
 			title: values.title,
 			text: values.text,
-			imageUrl: values.imageUrl,
+			imageUrl: values.imageUrl || null,
 			icon: values.icon,
 			display: values.display,
 			forExistingUsers: values.forExistingUsers,
--- a/packages/backend/src/core/QueueModule.ts
+++ b/packages/backend/src/core/QueueModule.ts
@ -18,6 +18,7 @@ import {
 	SystemWebhookDeliverJobData,
 } from '../queue/types.js';
 import type { Provider } from '@nestjs/common';
+import { mActiveJobs, mDelayedJobs, mJobReceivedCounter, mWaitingJobs } from '@/queue/metrics.js';

 export type SystemQueue = Bull.Queue<Record<string, unknown>>;
 export type EndedPollNotificationQueue = Bull.Queue<EndedPollNotificationJobData>;
@ -29,57 +30,73 @@ export type ObjectStorageQueue = Bull.Queue;
 export type UserWebhookDeliverQueue = Bull.Queue<UserWebhookDeliverJobData>;
 export type SystemWebhookDeliverQueue = Bull.Queue<SystemWebhookDeliverJobData>;

+function withMetrics<T>(queue: Bull.Queue<T>): Bull.Queue<T> {
+	if (process.env.NODE_ENV !== 'test') {
+		setInterval(async () => {
+			mActiveJobs?.set({ queue: queue.name }, await queue.getActiveCount());
+			mDelayedJobs?.set({ queue: queue.name }, await queue.getDelayedCount());
+			mWaitingJobs?.set({ queue: queue.name }, await queue.getWaitingCount());
+		}, 2000);
+
+		queue.on('waiting', () => {
+			mJobReceivedCounter?.inc({ queue: queue.name });
+		});
+	}
+	
+	return queue;
+}
+
 const $system: Provider = {
 	provide: 'queue:system',
-	useFactory: (config: Config) => new Bull.Queue(QUEUE.SYSTEM, baseQueueOptions(config, QUEUE.SYSTEM)),
+	useFactory: (config: Config) => withMetrics(new Bull.Queue(QUEUE.SYSTEM, baseQueueOptions(config, QUEUE.SYSTEM))),
 	inject: [DI.config],
 };

 const $endedPollNotification: Provider = {
 	provide: 'queue:endedPollNotification',
-	useFactory: (config: Config) => new Bull.Queue(QUEUE.ENDED_POLL_NOTIFICATION, baseQueueOptions(config, QUEUE.ENDED_POLL_NOTIFICATION)),
+	useFactory: (config: Config) => withMetrics(new Bull.Queue(QUEUE.ENDED_POLL_NOTIFICATION, baseQueueOptions(config, QUEUE.ENDED_POLL_NOTIFICATION))),
 	inject: [DI.config],
 };

 const $deliver: Provider = {
 	provide: 'queue:deliver',
-	useFactory: (config: Config) => new Bull.Queue(QUEUE.DELIVER, baseQueueOptions(config, QUEUE.DELIVER)),
+	useFactory: (config: Config) => withMetrics(new Bull.Queue(QUEUE.DELIVER, baseQueueOptions(config, QUEUE.DELIVER))),
 	inject: [DI.config],
 };

 const $inbox: Provider = {
 	provide: 'queue:inbox',
-	useFactory: (config: Config) => new Bull.Queue(QUEUE.INBOX, baseQueueOptions(config, QUEUE.INBOX)),
+	useFactory: (config: Config) => withMetrics(new Bull.Queue(QUEUE.INBOX, baseQueueOptions(config, QUEUE.INBOX))),
 	inject: [DI.config],
 };

 const $db: Provider = {
 	provide: 'queue:db',
-	useFactory: (config: Config) => new Bull.Queue(QUEUE.DB, baseQueueOptions(config, QUEUE.DB)),
+	useFactory: (config: Config) => withMetrics(new Bull.Queue(QUEUE.DB, baseQueueOptions(config, QUEUE.DB))),
 	inject: [DI.config],
 };

 const $relationship: Provider = {
 	provide: 'queue:relationship',
-	useFactory: (config: Config) => new Bull.Queue(QUEUE.RELATIONSHIP, baseQueueOptions(config, QUEUE.RELATIONSHIP)),
+	useFactory: (config: Config) => withMetrics(new Bull.Queue(QUEUE.RELATIONSHIP, baseQueueOptions(config, QUEUE.RELATIONSHIP))),
 	inject: [DI.config],
 };

 const $objectStorage: Provider = {
 	provide: 'queue:objectStorage',
-	useFactory: (config: Config) => new Bull.Queue(QUEUE.OBJECT_STORAGE, baseQueueOptions(config, QUEUE.OBJECT_STORAGE)),
+	useFactory: (config: Config) => withMetrics(new Bull.Queue(QUEUE.OBJECT_STORAGE, baseQueueOptions(config, QUEUE.OBJECT_STORAGE))),
 	inject: [DI.config],
 };

 const $userWebhookDeliver: Provider = {
 	provide: 'queue:userWebhookDeliver',
-	useFactory: (config: Config) => new Bull.Queue(QUEUE.USER_WEBHOOK_DELIVER, baseQueueOptions(config, QUEUE.USER_WEBHOOK_DELIVER)),
+	useFactory: (config: Config) => withMetrics(new Bull.Queue(QUEUE.USER_WEBHOOK_DELIVER, baseQueueOptions(config, QUEUE.USER_WEBHOOK_DELIVER))),
 	inject: [DI.config],
 };

 const $systemWebhookDeliver: Provider = {
 	provide: 'queue:systemWebhookDeliver',
-	useFactory: (config: Config) => new Bull.Queue(QUEUE.SYSTEM_WEBHOOK_DELIVER, baseQueueOptions(config, QUEUE.SYSTEM_WEBHOOK_DELIVER)),
+	useFactory: (config: Config) => withMetrics(new Bull.Queue(QUEUE.SYSTEM_WEBHOOK_DELIVER, baseQueueOptions(config, QUEUE.SYSTEM_WEBHOOK_DELIVER))),
 	inject: [DI.config],
 };

--- a/packages/backend/src/core/activitypub/ApInboxService.ts
+++ b/packages/backend/src/core/activitypub/ApInboxService.ts
@ -5,6 +5,7 @@

 import { Inject, Injectable } from '@nestjs/common';
 import { In } from 'typeorm';
+import * as prom from 'prom-client';
 import { DI } from '@/di-symbols.js';
 import type { Config } from '@/config.js';
 import { UserFollowingService } from '@/core/UserFollowingService.js';
@ -38,6 +39,13 @@ import { ApPersonService } from './models/ApPersonService.js';
 import { ApQuestionService } from './models/ApQuestionService.js';
 import type { Resolver } from './ApResolverService.js';
 import type { IAccept, IAdd, IAnnounce, IBlock, ICreate, IDelete, IFlag, IFollow, ILike, IObject, IReject, IRemove, IUndo, IUpdate, IMove, IPost } from './type.js';
+import { metricCounter } from '@/server/api/MetricsService.js';
+
+const mInboxReceived = metricCounter({
+	name: 'misskey_ap_inbox_received_total',
+	help: 'Total number of activities received by AP inbox',
+	labelNames: ['host', 'type'],
+});

@Injectable()
 export class ApInboxService {
@ -131,34 +139,49 @@ export class ApInboxService {
 		if (actor.isSuspended) return;

 		if (isCreate(activity)) {
+			mInboxReceived?.inc({ host: actor.host, type: 'create' });
 			return await this.create(actor, activity);
 		} else if (isDelete(activity)) {
+			mInboxReceived?.inc({ host: actor.host, type: 'delete' });
 			return await this.delete(actor, activity);
 		} else if (isUpdate(activity)) {
+			mInboxReceived?.inc({ host: actor.host, type: 'update' });
 			return await this.update(actor, activity);
 		} else if (isFollow(activity)) {
+			mInboxReceived?.inc({ host: actor.host, type: 'follow' });
 			return await this.follow(actor, activity);
 		} else if (isAccept(activity)) {
+			mInboxReceived?.inc({ host: actor.host, type: 'accept' });
 			return await this.accept(actor, activity);
 		} else if (isReject(activity)) {
+			mInboxReceived?.inc({ host: actor.host, type: 'reject' });
 			return await this.reject(actor, activity);
 		} else if (isAdd(activity)) {
+			mInboxReceived?.inc({ host: actor.host, type: 'add' });
 			return await this.add(actor, activity);
 		} else if (isRemove(activity)) {
+			mInboxReceived?.inc({ host: actor.host, type: 'remove' });
 			return await this.remove(actor, activity);
 		} else if (isAnnounce(activity)) {
+			mInboxReceived?.inc({ host: actor.host, type: 'announce' });
 			return await this.announce(actor, activity);
 		} else if (isLike(activity)) {
+			mInboxReceived?.inc({ host: actor.host, type: 'like' });
 			return await this.like(actor, activity);
 		} else if (isUndo(activity)) {
+			mInboxReceived?.inc({ host: actor.host, type: 'undo' });
 			return await this.undo(actor, activity);
 		} else if (isBlock(activity)) {
+			mInboxReceived?.inc({ host: actor.host, type: 'block' });
 			return await this.block(actor, activity);
 		} else if (isFlag(activity)) {
+			mInboxReceived?.inc({ host: actor.host, type: 'flag' });
 			return await this.flag(actor, activity);
 		} else if (isMove(activity)) {
+			mInboxReceived?.inc({ host: actor.host, type: 'move' });
 			return await this.move(actor, activity);
 		} else {
+			mInboxReceived?.inc({ host: actor.host, type: 'unknown' });
 			return `unrecognized activity type: ${activity.type}`;
 		}
 	}
@ -469,9 +492,9 @@ export class ApInboxService {
 			formerType = 'Note';
 		}

-		if (validPost.includes(formerType)) {
+		if (validPost?.includes(formerType)) {
 			return await this.deleteNote(actor, uri);
-		} else if (validActor.includes(formerType)) {
+		} else if (validActor?.includes(formerType)) {
 			return await this.deleteActor(actor, uri);
 		} else {
 			return `Unknown type ${formerType}`;
--- a/packages/backend/src/misc/log-sanitization.ts
+++ b/packages/backend/src/misc/log-sanitization.ts
@ -0,0 +1,42 @@
+/*
+ * SPDX-FileCopyrightText: syuilo and misskey-project and yumechi
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+import { aidRegExp } from "./id/aid.js";
+import { aidxRegExp } from "./id/aidx.js";
+
+export function sanitizeRequestURI(uri: string): string {
+    const vite = /^\/vite\/.+\.([a-z0-9]{1,4})$/;
+    const embed_vite = /^\/embed_vite\/.+\.([a-z0-9]{1,4})$/;
+
+    if (vite.test(uri)) {
+        return '[vite]';
+    }
+
+    if (embed_vite.test(uri)) {
+        return '[embed_vite]';
+    }
+
+    const uuid = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/g;
+    const username_local = /\/@\w+(\/|$)/;
+    const username_remote = /\/@\w+@[a-zA-Z0-9-.]+\.[a-zA-Z]{2,4}(\/|$)/;
+    const token = /=[0-9a-zA-Z]{16}/g;
+    const aidx = new RegExp(`/${aidxRegExp.source.replace(/^\^/, '').replace(/\$$/, '')}(\/|$)`, 'g');
+    const aid = new RegExp(`/${aidRegExp.source.replace(/^\^/, '').replace(/\$$/, '')}(\/|$)`, 'g');
+
+    return uri
+        .replace(/\/tags\/[^/]+/g, '/tags/[tag]')
+        .replace(/\/user-tags\/[^/]+/g, '/user-tags/[tag]')
+        .replace(/\/page\/[\w-]+/g, '/page/[page]')
+        .replace(/\/fluent-emoji\/[^/]+/g, '/fluent-emoji/[fluent-emoji]')
+        .replace(/\/twemoji\/[^/]+/g, '/twemoji/[twemoji]')
+        .replace(/\/twemoji-badge\/[^/]+/g, '/twemoji-badge/[twemoji-badge]')
+        .replace(aidx, '/[aidx]/')
+        .replace(aid, '/[aid]/')
+        .replace(token, '=[token]')
+        .replace(uuid, '[uuid]')
+        .replace(username_local, '/[user_local]/')
+        .replace(username_remote, '/[user_remote]/');
+}
+
--- a/packages/backend/src/postgres.ts
+++ b/packages/backend/src/postgres.ts
@ -7,6 +7,7 @@
 import pg from 'pg';
 import { DataSource, Logger } from 'typeorm';
 import * as highlight from 'cli-highlight';
+import { createHash } from 'crypto';
 import { entities as charts } from '@/core/chart/entities.js';

 import { MiAbuseUserReport } from '@/models/AbuseUserReport.js';
@ -82,6 +83,8 @@ import { MiReversiGame } from '@/models/ReversiGame.js';
 import { Config } from '@/config.js';
 import MisskeyLogger from '@/logger.js';
 import { bindThis } from '@/decorators.js';
+import { MemoryKVCache } from './misc/cache.js';
+import { metricCounter, metricHistogram } from './server/api/MetricsService.js';

 pg.types.setTypeParser(20, Number);

@ -89,7 +92,63 @@ export const dbLogger = new MisskeyLogger('db');

 const sqlLogger = dbLogger.createSubLogger('sql', 'gray');

+type QueryTagCache = {
+	join: string;
+	from: string;
+	hash: string;
+};
+
+function dedupConsecutive<T>(arr: T[]): T[] {
+	return arr.filter((v, i, a) => i === 0 || a[i - 1] !== v);
+}
+
+function simplifyIdentifiers(sql: string) {
+	return sql.replace(/"([a-zA-Z_]+)"/g, '$1');
+}
+
+function extractQueryTags(query: string): QueryTagCache {
+	const joins = query.matchAll(/(LEFT|RIGHT|INNER|OUTER) JOIN ([a-zA-Z0-9_"`.\s]+) ON/ig);
+	const froms = query.matchAll(/FROM ([a-zA-Z0-9_"`.]+)/ig);
+
+	const join = Array.from(joins).map(j => `${j[1]}:${simplifyIdentifiers(j[2])}`).join('|');
+	const from = dedupConsecutive(Array.from(froms).map(f => simplifyIdentifiers(f[1]))).join('|');
+
+	// this is not for security just to reduce metrics size just for confirmation in local environment
+	// all the code is public there is nothing secret in the query itself
+	const hash = createHash('md5');
+	hash.update(query);
+
+	return {
+		join,
+		from,
+		hash: hash.digest('hex'),
+	};
+}
+
+const mQueryCounter = metricCounter({
+	name: 'misskey_postgres_query_total',
+	help: 'Total queries to postgres',
+	labelNames: ['join', 'from', 'hash'],
+});
+
+const mQueryErrorCounter = metricCounter({
+	name: 'misskey_postgres_query_error_total',
+	help: 'Total errors in queries to postgres',
+	labelNames: ['join', 'from', 'hash'],
+});
+
+const mSlowQueryHisto = metricHistogram({
+	name: 'misskey_postgres_query_slow_duration_seconds',
+	help: 'Duration of slow queries to postgres',
+	labelNames: ['join', 'from', 'hash'],
+	buckets: [0.1, 0.5, 1, 2, 5, 10, 30, 300],
+});
+
 class MyCustomLogger implements Logger {
+	constructor(private metricOnly = true) {}
+
+	private queryHashCache = new MemoryKVCache<QueryTagCache>(1000 * 60 * 5); // 5m
+
 	@bindThis
 	private highlight(sql: string) {
 		return highlight.highlight(sql, {
@ -97,33 +156,69 @@ class MyCustomLogger implements Logger {
 		});
 	}

+	@bindThis
+	private getQueryTags(query: string): QueryTagCache {
+		const existing = this.queryHashCache.get(query);
+		if (existing) {
+			return existing;
+		}
+
+		const result = extractQueryTags(query);
+		this.queryHashCache.set(query, result);
+		return result;
+	}
+
 	@bindThis
 	public logQuery(query: string, parameters?: any[]) {
+		mQueryCounter?.inc(this.getQueryTags(query));
+
+		if (this.metricOnly) {
+			return;
+		}
 		sqlLogger.info(this.highlight(query).substring(0, 100));
 	}

 	@bindThis
 	public logQueryError(error: string, query: string, parameters?: any[]) {
+		mQueryErrorCounter?.inc(this.getQueryTags(query));
+
+		if (this.metricOnly) {
+			return;
+		}
 		sqlLogger.error(this.highlight(query));
 	}

 	@bindThis
 	public logQuerySlow(time: number, query: string, parameters?: any[]) {
+		mSlowQueryHisto?.observe(this.getQueryTags(query), time);
+
+		if (this.metricOnly) {
+			return;
+		}
 		sqlLogger.warn(this.highlight(query));
 	}

 	@bindThis
 	public logSchemaBuild(message: string) {
+		if (this.metricOnly) {
+			return;
+		}
 		sqlLogger.info(message);
 	}

 	@bindThis
 	public log(message: string) {
+		if (this.metricOnly) {
+			return;
+		}
 		sqlLogger.info(message);
 	}

 	@bindThis
 	public logMigration(message: string) {
+		if (this.metricOnly) {
+			return;
+		}
 		sqlLogger.info(message);
 	}
 }
@ -203,7 +298,7 @@ export const entities = [

 const log = process.env.NODE_ENV !== 'production';

-export function createPostgresDataSource(config: Config) {
+export function createPostgresDataSource(config: Config, isMain = false) {
 	return new DataSource({
 		type: 'postgres',
 		host: config.db.host,
@ -246,9 +341,9 @@ export function createPostgresDataSource(config: Config) {
 				db: config.redis.db ?? 0,
 			},
 		} : false,
-		logging: log,
-		logger: log ? new MyCustomLogger() : undefined,
-		maxQueryExecutionTime: 1000,
+		logging: true,
+		logger: new MyCustomLogger(!log),
+		maxQueryExecutionTime: 500,
 		entities: entities,
 		migrations: ['../../migration/*.js'],
 	});
--- a/packages/backend/src/queue/QueueProcessorService.ts
+++ b/packages/backend/src/queue/QueueProcessorService.ts
@ -45,6 +45,7 @@ import { CleanProcessorService } from './processors/CleanProcessorService.js';
 import { AggregateRetentionProcessorService } from './processors/AggregateRetentionProcessorService.js';
 import { QueueLoggerService } from './QueueLoggerService.js';
 import { QUEUE, baseQueueOptions } from './const.js';
+import { mStalledWorkerCounter } from './metrics.js';

 // ref. https://github.com/misskey-dev/misskey/pull/7635#issue-971097019
 function httpRelatedBackoff(attemptsMade: number) {
@ -194,7 +195,10 @@ export class QueueProcessorService implements OnApplicationShutdown {
 					}
 				})
 				.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
-				.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
+				.on('stalled', (jobId) => {
+					mStalledWorkerCounter?.inc({ queue: QUEUE.SYSTEM });
+					logger.warn(`stalled id=${jobId}`);
+				});
 		}
 		//#endregion

@ -251,7 +255,10 @@ export class QueueProcessorService implements OnApplicationShutdown {
 					}
 				})
 				.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
-				.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
+				.on('stalled', (jobId) => {
+					mStalledWorkerCounter?.inc({ queue: QUEUE.DB });
+					logger.warn(`stalled id=${jobId}`);
+				});
 		}
 		//#endregion

@ -291,7 +298,10 @@ export class QueueProcessorService implements OnApplicationShutdown {
 					}
 				})
 				.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
-				.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
+				.on('stalled', (jobId) => {
+					mStalledWorkerCounter?.inc({ queue: QUEUE.DELIVER });
+					logger.warn(`stalled id=${jobId}`);
+				});
 		}
 		//#endregion

@ -331,7 +341,10 @@ export class QueueProcessorService implements OnApplicationShutdown {
 					}
 				})
 				.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
-				.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
+				.on('stalled', (jobId) => {
+					mStalledWorkerCounter?.inc({ queue: QUEUE.INBOX });
+					logger.warn(`stalled id=${jobId}`);
+				});
 		}
 		//#endregion

@ -371,7 +384,10 @@ export class QueueProcessorService implements OnApplicationShutdown {
 					}
 				})
 				.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
-				.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
+				.on('stalled', (jobId) => {
+					mStalledWorkerCounter?.inc({ queue: QUEUE.USER_WEBHOOK_DELIVER });
+					logger.warn(`stalled id=${jobId}`);
+				});
 		}
 		//#endregion

@ -411,7 +427,10 @@ export class QueueProcessorService implements OnApplicationShutdown {
 					}
 				})
 				.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
-				.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
+				.on('stalled', (jobId) => {
+					mStalledWorkerCounter?.inc({ queue: QUEUE.SYSTEM_WEBHOOK_DELIVER });
+					logger.warn(`stalled id=${jobId}`);
+				});
 		}
 		//#endregion

@ -458,7 +477,10 @@ export class QueueProcessorService implements OnApplicationShutdown {
 					}
 				})
 				.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
-				.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
+				.on('stalled', (jobId) => {
+					mStalledWorkerCounter?.inc({ queue: QUEUE.RELATIONSHIP });
+					logger.warn(`stalled id=${jobId}`);
+				});
 		}
 		//#endregion

@ -499,7 +521,10 @@ export class QueueProcessorService implements OnApplicationShutdown {
 					}
 				})
 				.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
-				.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
+				.on('stalled', (jobId) => {
+					mStalledWorkerCounter?.inc({ queue: QUEUE.OBJECT_STORAGE });
+					logger.warn(`stalled id=${jobId}`);
+				});
 		}
 		//#endregion

--- a/packages/backend/src/queue/metrics.ts
+++ b/packages/backend/src/queue/metrics.ts
@ -0,0 +1,31 @@
+import { metricCounter, metricGauge } from '@/server/api/MetricsService.js';
+
+export const mJobReceivedCounter = metricCounter({
+    name: 'misskey_queue_jobs_received_total',
+    help: 'Total number of jobs received by queue',
+    labelNames: ['queue'],
+});
+
+export const mActiveJobs = metricGauge({
+    name: 'misskey_queue_active_jobs',
+    help: 'Number of active jobs in queue',
+    labelNames: ['queue'],
+});
+
+export const mDelayedJobs = metricGauge({
+    name: 'misskey_queue_delayed_jobs',
+    help: 'Number of delayed jobs in queue',
+    labelNames: ['queue'],
+});
+
+export const mWaitingJobs = metricGauge({
+    name: 'misskey_queue_waiting_jobs',
+    help: 'Number of waiting jobs in queue',
+    labelNames: ['queue'],
+});
+
+export const mStalledWorkerCounter = metricCounter({
+    name: 'misskey_queue_stalled_workers_total',
+    help: 'Total number of stalled workers',
+    labelNames: ['queue'],
+});
--- a/packages/backend/src/queue/processors/InboxProcessorService.ts
+++ b/packages/backend/src/queue/processors/InboxProcessorService.ts
@ -7,6 +7,7 @@ import { URL } from 'node:url';
 import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
 import httpSignature from '@peertube/http-signature';
 import * as Bull from 'bullmq';
+import * as prom from 'prom-client';
 import type Logger from '@/logger.js';
 import { FederatedInstanceService } from '@/core/FederatedInstanceService.js';
 import { FetchInstanceMetadataService } from '@/core/FetchInstanceMetadataService.js';
@ -31,12 +32,44 @@ import { MiMeta } from '@/models/Meta.js';
 import { DI } from '@/di-symbols.js';
 import { QueueLoggerService } from '../QueueLoggerService.js';
 import type { InboxJobData } from '../types.js';
+import { metricCounter, metricHistogram } from '@/server/api/MetricsService.js';

 type UpdateInstanceJob = {
 	latestRequestReceivedAt: Date,
 	shouldUnsuspend: boolean,
 };

+const mIncomingApProcessingTime = metricHistogram({
+	name: 'misskey_incoming_ap_processing_time',
+	help: 'Incoming AP processing time in seconds',
+	labelNames: ['incoming_host', 'incoming_type', 'success'],
+	buckets: [0.01, 0.1, 0.5, 1, 5, 10, 30, 60, 300, 1800],
+});
+
+const mIncomingApEvent = metricCounter({
+	name: 'misskey_incoming_ap_event',
+	help: 'Incoming AP event',
+	labelNames: ['incoming_host', 'incoming_type'],
+});
+
+const mIncomingApEventAccepted = metricCounter({
+	name: 'misskey_incoming_ap_event_accepted',
+	help: 'Incoming AP event accepted',
+	labelNames: ['incoming_host', 'incoming_type'],
+});
+
+const mIncomingApReject = metricCounter({
+	name: 'misskey_incoming_ap_reject',
+	help: 'Incoming AP reject',
+	labelNames: ['incoming_host', 'incoming_type', 'reason'],
+});
+
+const mincomingApProcessingError = metricCounter({
+	name: 'misskey_incoming_ap_processing_error',
+	help: 'Incoming AP processing error',
+	labelNames: ['incoming_host', 'incoming_type'],
+});
+
@Injectable()
 export class InboxProcessorService implements OnApplicationShutdown {
 	private logger: Logger;
@ -66,7 +99,6 @@ export class InboxProcessorService implements OnApplicationShutdown {
 	public async process(job: Bull.Job<InboxJobData>): Promise<string> {
 		const signature = job.data.signature;	// HTTP-signature
 		let activity = job.data.activity;
-
 		//#region Log
 		const info = Object.assign({}, activity);
 		delete info['@context'];
@ -75,12 +107,34 @@ export class InboxProcessorService implements OnApplicationShutdown {

 		const host = this.utilityService.toPuny(new URL(signature.keyId).hostname);

+		const incCounter = <T extends 'incoming_host' | 'incoming_type', U>(counter: prom.Counter<T> | null, addn_labels: U) => {
+			if (Array.isArray(activity.type)) {
+				for (const t of activity.type) {
+					counter?.inc({ incoming_host: host.toString(), incoming_type: t, ...addn_labels });
+				}
+			} else {
+				counter?.inc({ incoming_host: host.toString(), incoming_type: activity.type ?? 'unknown', ...addn_labels });
+			}
+		};
+
+		const observeHistogram = <T extends 'incoming_host' | 'incoming_type', U>(histogram: prom.Histogram<T> | null, addn_labels: U, value: number) => {
+			if (Array.isArray(activity.type)) {
+				for (const t of activity.type) {
+					histogram?.observe({ incoming_host: host.toString(), incoming_type: t, ...addn_labels }, value);
+				}
+			} else {
+				histogram?.observe({ incoming_host: host.toString(), incoming_type: activity.type ?? 'unknown', ...addn_labels }, value);
+			}
+		};
+
 		if (!this.utilityService.isFederationAllowedHost(host)) {
+			incCounter(mIncomingApReject, { reason: 'host_not_allowed' });
 			return `Blocked request: ${host}`;
 		}

 		const keyIdLower = signature.keyId.toLowerCase();
 		if (keyIdLower.startsWith('acct:')) {
+			incCounter(mIncomingApReject, { reason: 'keyid_acct' });
 			return `Old keyId is no longer supported. ${keyIdLower}`;
 		}

@ -98,6 +152,7 @@ export class InboxProcessorService implements OnApplicationShutdown {
 				// 対象が4xxならスキップ
 				if (err instanceof StatusError) {
 					if (!err.isRetryable) {
+						incCounter(mIncomingApReject, { reason: 'actor_key_unresolvable' });
 						throw new Bull.UnrecoverableError(`skip: Ignored deleted actors on both ends ${activity.actor} - ${err.statusCode}`);
 					}
 					throw new Error(`Error in actor ${activity.actor} - ${err.statusCode}`);
@ -107,11 +162,13 @@ export class InboxProcessorService implements OnApplicationShutdown {

 		// それでもわからなければ終了
 		if (authUser == null) {
+			incCounter(mIncomingApReject, { reason: 'actor_unresolvable' });
 			throw new Bull.UnrecoverableError('skip: failed to resolve user');
 		}

 		// publicKey がなくても終了
 		if (authUser.key == null) {
+			incCounter(mIncomingApReject, { reason: 'publickey_unresolvable' });
 			throw new Bull.UnrecoverableError('skip: failed to resolve user publicKey');
 		}

@ -124,6 +181,7 @@ export class InboxProcessorService implements OnApplicationShutdown {
 			const ldSignature = activity.signature;
 			if (ldSignature) {
 				if (ldSignature.type !== 'RsaSignature2017') {
+					incCounter(mIncomingApReject, { reason: 'ld_signature_unsupported' });
 					throw new Bull.UnrecoverableError(`skip: unsupported LD-signature type ${ldSignature.type}`);
 				}

@ -137,10 +195,12 @@ export class InboxProcessorService implements OnApplicationShutdown {
 				// keyIdからLD-Signatureのユーザーを取得
 				authUser = await this.apDbResolverService.getAuthUserFromKeyId(ldSignature.creator);
 				if (authUser == null) {
+					incCounter(mIncomingApReject, { reason: 'ld_signature_user_unresolvable' });
 					throw new Bull.UnrecoverableError('skip: LD-Signatureのユーザーが取得できませんでした');
 				}

 				if (authUser.key == null) {
+					incCounter(mIncomingApReject, { reason: 'ld_signature_publickey_unavailable' });
 					throw new Bull.UnrecoverableError('skip: LD-SignatureのユーザーはpublicKeyを持っていませんでした');
 				}

@ -149,6 +209,7 @@ export class InboxProcessorService implements OnApplicationShutdown {
 				// LD-Signature検証
 				const verified = await jsonLd.verifyRsaSignature2017(activity, authUser.key.keyPem).catch(() => false);
 				if (!verified) {
+					incCounter(mIncomingApReject, { reason: 'ld_signature_verification_failed' });
 					throw new Bull.UnrecoverableError('skip: LD-Signatureの検証に失敗しました');
 				}

@ -171,14 +232,17 @@ export class InboxProcessorService implements OnApplicationShutdown {

 				// もう一度actorチェック
 				if (authUser.user.uri !== activity.actor) {
+					incCounter(mIncomingApReject, { reason: 'ld_signature_actor_mismatch' });
 					throw new Bull.UnrecoverableError(`skip: LD-Signature user(${authUser.user.uri}) !== activity.actor(${activity.actor})`);
 				}

 				const ldHost = this.utilityService.extractDbHost(authUser.user.uri);
 				if (!this.utilityService.isFederationAllowedHost(ldHost)) {
+					incCounter(mIncomingApReject, { reason: 'fed_host_not_allowed' });
 					throw new Bull.UnrecoverableError(`Blocked request: ${ldHost}`);
 				}
 			} else {
+				incCounter(mIncomingApReject, { reason: 'ld_signature_unavailable' });
 				throw new Bull.UnrecoverableError(`skip: http-signature verification failed and no LD-Signature. keyId=${signature.keyId}`);
 			}
 		}
@ -188,6 +252,7 @@ export class InboxProcessorService implements OnApplicationShutdown {
 			const signerHost = this.utilityService.extractDbHost(authUser.user.uri!);
 			const activityIdHost = this.utilityService.extractDbHost(activity.id);
 			if (signerHost !== activityIdHost) {
+				incCounter(mIncomingApReject, 'host_signature_mismatch');
 				throw new Bull.UnrecoverableError(`skip: signerHost(${signerHost}) !== activity.id host(${activityIdHost}`);
 			}
 		}
@ -215,7 +280,10 @@ export class InboxProcessorService implements OnApplicationShutdown {
 			this.fetchInstanceMetadataService.fetchInstanceMetadata(i);
 		});

+		incCounter(mIncomingApEvent, {});
+
 		// アクティビティを処理
+		const begin = +new Date();
 		try {
 			const result = await this.apInboxService.performActivity(authUser.user, activity);
 			if (result && !result.startsWith('ok')) {
@ -225,17 +293,26 @@ export class InboxProcessorService implements OnApplicationShutdown {
 		} catch (e) {
 			if (e instanceof IdentifiableError) {
 				if (e.id === '689ee33f-f97c-479a-ac49-1b9f8140af99') {
+					incCounter(mIncomingApReject, { reason: 'blocked_notes_with_prohibited_words' });
 					return 'blocked notes with prohibited words';
 				}
 				if (e.id === '85ab9bd7-3a41-4530-959d-f07073900109') {
+					incCounter(mIncomingApReject, { reason: 'actor_suspended' });
 					return 'actor has been suspended';
 				}
 				if (e.id === 'd450b8a9-48e4-4dab-ae36-f4db763fda7c') { // invalid Note
+					incCounter(mIncomingApReject, { reason: 'invalid_note' });
 					return e.message;
 				}
 			}
+			const end = +new Date();
+			observeHistogram(mIncomingApProcessingTime, { success: 'false' }, (end - begin) / 1000);
+			incCounter(mincomingApProcessingError, { reason: 'unknown' });
 			throw e;
 		}
+		observeHistogram(mIncomingApProcessingTime, { success: 'true' }, (+new Date() - begin) / 1000);
+		incCounter(mIncomingApEventAccepted, {});
+		
 		return 'ok';
 	}

--- a/packages/backend/src/server/ServerModule.ts
+++ b/packages/backend/src/server/ServerModule.ts
@ -47,6 +47,7 @@ import { RoleTimelineChannelService } from './api/stream/channels/role-timeline.
 import { ReversiChannelService } from './api/stream/channels/reversi.js';
 import { ReversiGameChannelService } from './api/stream/channels/reversi-game.js';
 import { SigninWithPasskeyApiService } from './api/SigninWithPasskeyApiService.js';
+import { MetricsService } from './api/MetricsService.js';

@Module({
 	imports: [
@ -94,6 +95,7 @@ import { SigninWithPasskeyApiService } from './api/SigninWithPasskeyApiService.j
 		UserListChannelService,
 		OpenApiServerService,
 		OAuth2ProviderService,
+		MetricsService,
 	],
 	exports: [
 		ServerService,
--- a/packages/backend/src/server/ServerService.ts
+++ b/packages/backend/src/server/ServerService.ts
@ -33,9 +33,83 @@ import { OpenApiServerService } from './api/openapi/OpenApiServerService.js';
 import { OAuth2ProviderService } from './oauth/OAuth2ProviderService.js';
 import { makeHstsHook } from './hsts.js';
 import { generateCSP } from './csp.js';
+import * as prom from 'prom-client';
+import { sanitizeRequestURI } from '@/misc/log-sanitization.js';
+import { metricCounter, metricGauge, metricHistogram, MetricsService } from './api/MetricsService.js';

 const _dirname = fileURLToPath(new URL('.', import.meta.url));

+function categorizeRequestPath(path: string): 'api' | 'health' | 'vite' | 'other' {
+	if (path === '/healthz') {
+		return 'health';
+	}
+
+	if (path.startsWith('/vite/') || path.startsWith('/embed_vite/')) {
+		return 'vite';
+	}
+
+	if (path === '/api' || path.startsWith('/api/')) {
+		return 'api';
+	} 
+
+	return 'other';
+}
+
+const mRequestTime = metricHistogram({
+	name: 'misskey_http_request_duration_seconds',
+	help: 'Duration of handling HTTP requests in seconds',
+	labelNames: ['host', 'cate', 'method', 'path'],
+	buckets: [0.001, 0.1, 0.5, 1, 2, 5],
+});
+
+const mRequestsReceived = metricCounter({
+	name: 'misskey_http_requests_received_total',
+	help: 'Total number of HTTP requests received',
+	labelNames: [],
+});
+
+const mNotFoundServed = metricCounter({
+	name: 'misskey_http_not_found_served_total',
+	help: 'Total number of HTTP 404 responses served',
+	labelNames: ['method', 'cate'],
+});
+
+const mMethodNotAllowedServed = metricCounter({
+	name: 'misskey_http_method_not_allowed_served_total',
+	help: 'Total number of HTTP 405 responses served',
+	labelNames: ['method', 'cate'],
+});
+
+const mTooManyRequestsServed = metricCounter({
+	name: 'misskey_http_too_many_requests_served_total',
+	help: 'Total number of HTTP 429 responses served',
+	labelNames: ['method', 'cate'],
+});
+
+const mAggregateRequestsServed = metricCounter({
+	name: 'misskey_http_requests_served_total',
+	help: 'Total number of HTTP requests served including invalid requests',
+	labelNames: ['host', 'cate', 'status'],
+});
+
+const mRequestsServedByPath = metricCounter({
+	name: 'misskey_http_requests_served_by_path',
+	help: 'Total number of HTTP requests served',
+	labelNames: ['host', 'cate', 'method', 'path', 'status'],
+});
+
+const mFatalErrorCount = metricCounter({
+	name: 'misskey_fatal_http_errors_total',
+	help: 'Total number of HTTP errors that propagate to the top level',
+	labelNames: ['host', 'cate', 'method', 'path'],
+});
+
+const mLastSuccessfulRequest = metricGauge({
+	name: 'misskey_http_last_successful_request_timestamp_seconds',
+	help: 'Unix Timestamp of the last successful HTTP request',
+	labelNames: [],
+});
+
@Injectable()
 export class ServerService implements OnApplicationShutdown {
 	private logger: Logger;
@ -70,6 +144,7 @@ export class ServerService implements OnApplicationShutdown {
 		private globalEventService: GlobalEventService,
 		private loggerService: LoggerService,
 		private oauth2ProviderService: OAuth2ProviderService,
+		private metricsService: MetricsService,
 	) {
 		this.logger = this.loggerService.getLogger('server', 'gray');
 	}
@ -82,6 +157,111 @@ export class ServerService implements OnApplicationShutdown {
 		});
 		this.#fastify = fastify;

+		if (this.config.prometheusMetrics?.enable) {
+			fastify.addHook('onRequest', (_request, reply, done) => {
+				reply.header('x-request-received', (+new Date()).toString());
+				mRequestsReceived?.inc();
+				done();
+			});
+
+			fastify.addHook('onError', (request, _reply, error, done) => {
+				const url = new URL(request.url, this.config.url);
+				const logPath = sanitizeRequestURI(url.pathname);
+				mFatalErrorCount?.inc({
+					host: request.hostname,
+					method: request.method,
+					path: logPath,
+					cate: categorizeRequestPath(logPath),
+				});
+				done();
+			});
+
+			fastify.addHook('onResponse', (request, reply, done) => {
+				const url = new URL(request.url, this.config.url);
+				const logPath = sanitizeRequestURI(url.pathname);
+				const cate = categorizeRequestPath(logPath);
+				const received = reply.getHeader('x-request-received') as string;
+
+				mAggregateRequestsServed?.inc({
+					host: request.hostname,
+					cate,
+					status: reply.statusCode,
+				});
+
+				if (reply.statusCode === 429) {
+					mTooManyRequestsServed?.inc({
+						method: request.method,
+						cate,
+					});
+
+					done();
+					return;
+				}
+
+				if (reply.statusCode === 404) {
+					mNotFoundServed?.inc({
+						method: request.method,
+						cate,
+					});
+
+					if (received) {
+						const duration = (+new Date()) - parseInt(received);
+						mRequestTime?.observe({
+							host: request.hostname,
+							method: request.method,
+							cate,
+						}, duration / 1000);
+					}
+
+					done();
+					return;
+				}
+
+				if (reply.statusCode === 405) {
+					mMethodNotAllowedServed?.inc({
+						method: request.method,
+						cate,
+					});
+					done();
+					return;
+				}
+
+				if (received) {
+					const duration = (+new Date()) - parseInt(received);
+
+					mRequestTime?.observe({
+						host: request.hostname,
+						method: request.method,
+						cate,
+						path: logPath,
+					}, duration / 1000);
+				}
+
+				if (logPath === '/metrics' || logPath === '/healthz') {
+					done();
+					return;
+				}
+
+				if (reply.statusCode <= 299) {
+					mLastSuccessfulRequest?.set(+new Date() / 1000);
+				}
+
+				mRequestsServedByPath?.inc({
+					host: request.hostname,
+					method: request.method,
+					path: logPath,
+					cate,
+					status: reply.statusCode >= 500 ? '5xx' :
+						reply.statusCode === 401 ? '401' :
+							reply.statusCode === 403 ? '403' :
+								reply.statusCode >= 400 ?
+									'4xx' : '2xx',
+				});
+
+				done();
+			});
+		}
+
 		// HSTS
 		if (this.config.url.startsWith('https') && !this.config.disableHsts) {
 			const preload = this.config.hstsPreload;
@ -253,6 +433,8 @@ export class ServerService implements OnApplicationShutdown {

 		fastify.register(this.clientServerService.createServer);

+		fastify.register(this.metricsService.createServer);
+
 		this.streamingApiServerService.attach(fastify.server);

 		fastify.server.on('error', err => {
--- a/packages/backend/src/server/api/AuthenticateService.ts
+++ b/packages/backend/src/server/api/AuthenticateService.ts
@ -5,6 +5,7 @@

 import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
 import { DI } from '@/di-symbols.js';
+import * as prom from 'prom-client';
 import type { AccessTokensRepository, AppsRepository, UsersRepository } from '@/models/_.js';
 import type { MiLocalUser } from '@/models/User.js';
 import type { MiAccessToken } from '@/models/AccessToken.js';
@ -13,6 +14,7 @@ import type { MiApp } from '@/models/App.js';
 import { CacheService } from '@/core/CacheService.js';
 import isNativeToken from '@/misc/is-native-token.js';
 import { bindThis } from '@/decorators.js';
+import { metricCounter } from './MetricsService.js';

 export class AuthenticationError extends Error {
 	constructor(message: string) {
@ -21,6 +23,12 @@ export class AuthenticationError extends Error {
 	}
 }

+const mAuthenticationFailureCounter = metricCounter({
+	name: 'misskey_authentication_failure_total',
+	help: 'Total number of authentication failures',
+	labelNames: ['cred_ty'],
+});
+
@Injectable()
 export class AuthenticateService implements OnApplicationShutdown {
 	private appCache: MemoryKVCache<MiApp>;
@ -51,6 +59,7 @@ export class AuthenticateService implements OnApplicationShutdown {
 				() => this.usersRepository.findOneBy({ token }) as Promise<MiLocalUser | null>);

 			if (user == null) {
+				mAuthenticationFailureCounter?.inc({ cred_ty: 'native' });
 				throw new AuthenticationError('user not found');
 			}

@ -65,6 +74,7 @@ export class AuthenticateService implements OnApplicationShutdown {
 			});

 			if (accessToken == null) {
+				mAuthenticationFailureCounter?.inc({ cred_ty: 'access_token' });
 				throw new AuthenticationError('invalid signature');
 			}

--- a/packages/backend/src/server/api/MetricsService.ts
+++ b/packages/backend/src/server/api/MetricsService.ts
@ -0,0 +1,76 @@
+import { Inject, Injectable } from "@nestjs/common";
+import * as prom from 'prom-client';
+import { DI } from '@/di-symbols.js';
+import type { Config } from '@/config.js';
+import { bindThis } from "@/decorators.js";
+import type { FastifyInstance, FastifyPluginOptions } from "fastify";
+
+export function metricGauge<K extends string>(conf: prom.GaugeConfiguration<K>) : prom.Gauge<K> | null {
+	if (!process.env.RUN_MODE) {
+		return null;
+	}
+
+	return new prom.Gauge(conf);
+}
+
+export function metricCounter<K extends string>(conf: prom.CounterConfiguration<K>) : prom.Counter<K> | null {
+	if (!process.env.RUN_MODE) {
+		return null;
+	}
+
+	return new prom.Counter(conf);
+}
+
+export function metricHistogram<K extends string>(conf: prom.HistogramConfiguration<K>) : prom.Histogram<K> | null {
+	if (!process.env.RUN_MODE) {
+		return null;
+	}
+
+	return new prom.Histogram(conf);
+}
+
+/*
+ * SPDX-FileCopyrightText: syuilo and misskey-project and yumechi
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+@Injectable()
+export class MetricsService {
+    constructor(
+		@Inject(DI.config)
+		private config: Config
+     ) {}
+
+    @bindThis
+    public createServer(fastify: FastifyInstance, options: FastifyPluginOptions, done: (err?: Error) => void) {
+        if (this.config.prometheusMetrics?.enable) {
+			const token = this.config.prometheusMetrics.scrapeToken;
+			fastify.get('/metrics', async (request, reply) => {
+				if (token) {
+					const bearer = request.headers.authorization;
+
+					if (!bearer) {
+						reply.code(401);
+						return;
+					}
+
+					const [type, t] = bearer.split(' ');
+
+					if (type !== 'Bearer' || t !== token) {
+						reply.code(403);
+						return;
+					}
+				}
+				
+				try {
+					reply.header('Content-Type', prom.register.contentType);
+					reply.send(await prom.register.metrics());
+				} catch (err) {
+					reply.code(500);
+				}
+			});
+		}
+
+		done();
+    }
+}
--- a/packages/backend/src/server/api/SigninApiService.ts
+++ b/packages/backend/src/server/api/SigninApiService.ts
@ -6,6 +6,7 @@
 import { Inject, Injectable } from '@nestjs/common';
 import bcrypt from 'bcryptjs';
 import { IsNull } from 'typeorm';
+import * as prom from 'prom-client';
 import * as Misskey from 'misskey-js';
 import { DI } from '@/di-symbols.js';
 import type {
@ -28,6 +29,13 @@ import { RateLimiterService } from './RateLimiterService.js';
 import { SigninService } from './SigninService.js';
 import type { AuthenticationResponseJSON } from '@simplewebauthn/types';
 import type { FastifyReply, FastifyRequest } from 'fastify';
+import { metricCounter } from './MetricsService.js';
+
+const mSigninFailureCounter = metricCounter({
+	name: 'misskey_misskey_signin_failure',
+	help: 'The number of failed sign-ins',
+	labelNames: ['reason'],
+});

@Injectable()
 export class SigninApiService {
@ -93,6 +101,7 @@ export class SigninApiService {
 		// not more than 1 attempt per second and not more than 10 attempts per hour
 			await this.rateLimiterService.limit({ key: 'signin', duration: 60 * 60 * 1000, max: 10, minInterval: 1000 }, getIpHash(request.ip));
 		} catch (err) {
+			mSigninFailureCounter?.inc({ reason: 'rate_limit' });
 			reply.code(429);
 			return {
 				error: {
@ -104,11 +113,13 @@ export class SigninApiService {
 		}

 		if (typeof username !== 'string') {
+			mSigninFailureCounter?.inc({ reason: 'bad_form' });
 			reply.code(400);
 			return;
 		}

 		if (token != null && typeof token !== 'string') {
+			mSigninFailureCounter?.inc({ reason: 'bad_form' });
 			reply.code(400);
 			return;
 		}
@ -120,12 +131,14 @@ export class SigninApiService {
 		}) as MiLocalUser;

 		if (user == null) {
+			mSigninFailureCounter?.inc({ reason: 'user_not_found' });
 			return error(404, {
 				id: '6cc579cc-885d-43d8-95c2-b8c7fc963280',
 			});
 		}

 		if (user.isSuspended) {
+			mSigninFailureCounter?.inc({ reason: 'user_suspended' });
 			return error(403, {
 				id: 'e03a5f46-d309-4865-9b69-56282d94e1eb',
 			});
@ -150,6 +163,7 @@ export class SigninApiService {
 		}

 		if (typeof password !== 'string') {
+			mSigninFailureCounter?.inc({ reason: 'bad_form' });
 			reply.code(400);
 			return;
 		}
@ -167,6 +181,7 @@ export class SigninApiService {
 				success: false,
 			});

+			mSigninFailureCounter?.inc({ reason: failure?.id ?? `unknown_error_${status ?? 500}` });
 			return error(status ?? 500, failure ?? { id: '4e30e80c-e338-45a0-8c8f-44455efa3b76' });
 		};

@ -174,30 +189,35 @@ export class SigninApiService {
 			if (process.env.NODE_ENV !== 'test') {
 				if (this.meta.enableHcaptcha && this.meta.hcaptchaSecretKey) {
 					await this.captchaService.verifyHcaptcha(this.meta.hcaptchaSecretKey, body['hcaptcha-response']).catch(err => {
+						mSigninFailureCounter?.inc({ reason: 'captcha_verification_failed_hcaptcha' });
 						throw new FastifyReplyError(400, err);
 					});
 				}

 				if (this.meta.enableMcaptcha && this.meta.mcaptchaSecretKey && this.meta.mcaptchaSitekey && this.meta.mcaptchaInstanceUrl) {
 					await this.captchaService.verifyMcaptcha(this.meta.mcaptchaSecretKey, this.meta.mcaptchaSitekey, this.meta.mcaptchaInstanceUrl, body['m-captcha-response']).catch(err => {
+						mSigninFailureCounter?.inc({ reason: 'captcha_verification_failed_mcaptcha' });
 						throw new FastifyReplyError(400, err);
 					});
 				}

 				if (this.meta.enableRecaptcha && this.meta.recaptchaSecretKey) {
 					await this.captchaService.verifyRecaptcha(this.meta.recaptchaSecretKey, body['g-recaptcha-response']).catch(err => {
+						mSigninFailureCounter?.inc({ reason: 'captcha_verification_failed_recaptcha' });
 						throw new FastifyReplyError(400, err);
 					});
 				}

 				if (this.meta.enableTurnstile && this.meta.turnstileSecretKey) {
 					await this.captchaService.verifyTurnstile(this.meta.turnstileSecretKey, body['turnstile-response']).catch(err => {
+						mSigninFailureCounter?.inc({ reason: 'captcha_verification_failed_turnstile' });
 						throw new FastifyReplyError(400, err);
 					});
 				}

 				if (this.meta.enableTestcaptcha) {
 					await this.captchaService.verifyTestcaptcha(body['testcaptcha-response']).catch(err => {
+						mSigninFailureCounter?.inc({ reason: 'captcha_verification_failed_testcaptcha' });
 						throw new FastifyReplyError(400, err);
 					});
 				}
--- a/packages/backend/src/server/api/endpoints/admin/announcements/create.ts
+++ b/packages/backend/src/server/api/endpoints/admin/announcements/create.ts
@ -55,7 +55,7 @@ export const paramDef = {
 	properties: {
 		title: { type: 'string', minLength: 1 },
 		text: { type: 'string', minLength: 1 },
-		imageUrl: { type: 'string', nullable: true, minLength: 1 },
+		imageUrl: { type: 'string', nullable: true, minLength: 0 },
 		icon: { type: 'string', enum: ['info', 'warning', 'error', 'success'], default: 'info' },
 		display: { type: 'string', enum: ['normal', 'banner', 'dialog'], default: 'normal' },
 		forExistingUsers: { type: 'boolean', default: false },
@ -76,7 +76,8 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
 				updatedAt: null,
 				title: ps.title,
 				text: ps.text,
-				imageUrl: ps.imageUrl,
+				/* eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- 空の文字列の場合、nullを渡すようにするため */
+				imageUrl: ps.imageUrl || null,
 				icon: ps.icon,
 				display: ps.display,
 				forExistingUsers: ps.forExistingUsers,
--- a/packages/backend/test-server/entry.ts
+++ b/packages/backend/test-server/entry.ts
@ -5,7 +5,6 @@ import { NestFactory } from '@nestjs/core';
 import { MainModule } from '@/MainModule.js';
 import { ServerService } from '@/server/ServerService.js';
 import { loadConfig } from '@/config.js';
-import { NestLogger } from '@/NestLogger.js';
 import { INestApplicationContext } from '@nestjs/common';

 const config = loadConfig();
@ -22,10 +21,8 @@ let serverService: ServerService;
 async function launch() {
 	await killTestServer();

-	console.log('starting application...');
-
 	app = await NestFactory.createApplicationContext(MainModule, {
-		logger: new NestLogger(),
+		logger: ["debug", "log", "error", "warn", "verbose"],
 	});
 	serverService = app.get(ServerService);
 	await serverService.launch();
@ -84,7 +81,7 @@ async function startControllerEndpoints(port = config.port + 1000) {
 		console.log('starting application...');

 		app = await NestFactory.createApplicationContext(MainModule, {
-			logger: new NestLogger(),
+			logger: ["debug", "log", "error", "warn", "verbose"],
 		});
 		serverService = app.get(ServerService);
 		await serverService.launch();
--- a/packages/frontend/.storybook/fakes.ts
+++ b/packages/frontend/.storybook/fakes.ts
@ -167,7 +167,7 @@ export function federationInstance(): entities.FederationInstance {
 		notesCount: 20,
 		followingCount: 5,
 		followersCount: 15,
-		isNotResponding: false,
+		: false,
 		isSuspended: false,
 		suspensionState: 'none',
 		isBlocked: false,
--- a/packages/misskey-js/etc/misskey-js.api.md
+++ b/packages/misskey-js/etc/misskey-js.api.md
@ -1812,10 +1812,6 @@ declare namespace entities {
        MeDetailed,
        UserDetailed,
        User,
-        UserWebhookBody,
-        UserWebhookNoteBody,
-        UserWebhookUserBody,
-        UserWebhookReactionBody,
        UserList,
        Ad,
        Announcement,
@ -3414,18 +3410,6 @@ type UsersShowResponse = operations['users___show']['responses']['200']['content
 // @public (undocumented)
 type UsersUpdateMemoRequest = operations['users___update-memo']['requestBody']['content']['application/json'];

-// @public (undocumented)
-type UserWebhookBody = components['schemas']['UserWebhookBody'];
-
-// @public (undocumented)
-type UserWebhookNoteBody = components['schemas']['UserWebhookNoteBody'];
-
-// @public (undocumented)
-type UserWebhookReactionBody = components['schemas']['UserWebhookReactionBody'];
-
-// @public (undocumented)
-type UserWebhookUserBody = components['schemas']['UserWebhookUserBody'];
-
 // Warnings were encountered during analysis:
 //
 // src/entities.ts:50:2 - (ae-forgotten-export) The symbol "ModerationLogPayloads" needs to be exported by the entry point index.d.ts
--- a/packages/misskey-js/src/autogen/models.ts
+++ b/packages/misskey-js/src/autogen/models.ts
@ -7,10 +7,6 @@ export type UserDetailedNotMe = components['schemas']['UserDetailedNotMe'];
 export type MeDetailed = components['schemas']['MeDetailed'];
 export type UserDetailed = components['schemas']['UserDetailed'];
 export type User = components['schemas']['User'];
-export type UserWebhookBody = components['schemas']['UserWebhookBody'];
-export type UserWebhookNoteBody = components['schemas']['UserWebhookNoteBody'];
-export type UserWebhookUserBody = components['schemas']['UserWebhookUserBody'];
-export type UserWebhookReactionBody = components['schemas']['UserWebhookReactionBody'];
 export type UserList = components['schemas']['UserList'];
 export type Ad = components['schemas']['Ad'];
 export type Announcement = components['schemas']['Announcement'];
--- a/packages/misskey-js/src/autogen/types.ts
+++ b/packages/misskey-js/src/autogen/types.ts
@ -4022,32 +4022,6 @@ export type components = {
    MeDetailed: components['schemas']['UserLite'] & components['schemas']['UserDetailedNotMeOnly'] & components['schemas']['MeDetailedOnly'];
    UserDetailed: components['schemas']['UserDetailedNotMe'] | components['schemas']['MeDetailed'];
    User: components['schemas']['UserLite'] | components['schemas']['UserDetailed'];
-    UserWebhookBody: OneOf<[{
-      note: components['schemas']['Note'];
-    }, {
-      user: components['schemas']['UserLite'];
-    }, {
-      note: components['schemas']['Note'];
-      reaction: {
-        id: string;
-        user: components['schemas']['UserLite'];
-        reaction: string;
-      };
-    }]>;
-    UserWebhookNoteBody: {
-      note: components['schemas']['Note'];
-    };
-    UserWebhookUserBody: {
-      user: components['schemas']['UserLite'];
-    };
-    UserWebhookReactionBody: {
-      note: components['schemas']['Note'];
-      reaction: {
-        id: string;
-        user: components['schemas']['UserLite'];
-        reaction: string;
-      };
-    };
    UserList: {
      /**
       * Format: id
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@ -12,6 +12,9 @@ importers:

  .:
    dependencies:
+      cross-env:
+        specifier: 7.0.3
+        version: 7.0.3
      cssnano:
        specifier: 6.1.2
        version: 6.1.2(postcss@8.4.49)
@ -62,9 +65,6 @@ importers:
      '@typescript-eslint/parser':
        specifier: 7.17.0
        version: 7.17.0(eslint@9.14.0)(typescript@5.6.3)
-      cross-env:
-        specifier: 7.0.3
-        version: 7.0.3
      cypress:
        specifier: 13.15.2
        version: 13.15.2
@ -350,6 +350,9 @@ importers:
      probe-image-size:
        specifier: 7.2.3
        version: 7.2.3
+      prom-client:
+        specifier: ^15.1.3
+        version: 15.1.3
      promise-limit:
        specifier: 2.7.0
        version: 2.7.0
@ -5252,6 +5255,9 @@ packages:
    resolution: {integrity: sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==}
    engines: {node: '>=8'}

+  bintrees@1.0.2:
+    resolution: {integrity: sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==}
+
  blob-util@2.0.2:
    resolution: {integrity: sha512-T7JQa+zsXXEa6/8ZhHcQEW1UFfVM49Ts65uBkFL6fz2QmrElqmbajIDJvuA0tEhRe5eIjpV9ZF+0RfZR9voJFQ==}

@ -9329,6 +9335,10 @@ packages:
    resolution: {integrity: sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==}
    engines: {node: '>=0.4.0'}

+  prom-client@15.1.3:
+    resolution: {integrity: sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==}
+    engines: {node: ^16 || ^18 || >=20}
+
  promise-limit@2.7.0:
    resolution: {integrity: sha512-7nJ6v5lnJsXwGprnGXga4wx6d1POjvi5Qmf1ivTRxTjH4Z/9Czja/UCMLVmB9N93GeWOU93XaFaEt6jbuoagNw==}

@ -10345,6 +10355,9 @@ packages:
    resolution: {integrity: sha512-+HRtZ40Vc+6YfCDWCeAsixwxJgMbPY4HHuTgzPYH3JXvqHWUlsCfy+ylXlAKhFNcuLp4xVeWeFBUhDk+7KYUvQ==}
    engines: {node: '>=14.16'}

+  tdigest@0.1.2:
+    resolution: {integrity: sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==}
+
  terser@5.36.0:
    resolution: {integrity: sha512-IYV9eNMuFAV4THUspIRXkLakHnV6XO7FEdtKjf/mDyrnqUg9LnlOn6/RwRvM9SZjR4GUq8Nk8zj67FzVARr74w==}
    engines: {node: '>=10'}
@ -16086,6 +16099,8 @@ snapshots:

  binary-extensions@2.2.0: {}

+  bintrees@1.0.2: {}
+
  blob-util@2.0.2: {}

  bluebird@3.7.2: {}
@ -21097,6 +21112,11 @@ snapshots:
  progress@2.0.3:
    optional: true

+  prom-client@15.1.3:
+    dependencies:
+      '@opentelemetry/api': 1.9.0
+      tdigest: 0.1.2
+
  promise-limit@2.7.0: {}

  promise-polyfill@8.3.0: {}
@ -22280,6 +22300,10 @@ snapshots:
    dependencies:
      execa: 6.1.0

+  tdigest@0.1.2:
+    dependencies:
+      bintrees: 1.0.2
+
  terser@5.36.0:
    dependencies:
      '@jridgewell/source-map': 0.3.6
--- a/yume-mods/etc/config.alloy
+++ b/yume-mods/etc/config.alloy
@ -0,0 +1,18 @@
+prometheus.scrape "yumechinokuni" {
+    targets = [
+        {
+            __address__ = "mi.yumechi.jp:443",
+            __scheme__ = "https",
+            environment = "prod",
+            hostname = "mi.yumechi.jp",
+        },
+        {
+            __address__ = "test0.mi.yumechi.jp:443",
+            __scheme__ = "https",
+            environment = "test",
+            hostname = "test0.mi.yumechi.jp",
+        }
+    ]
+
+    forward_to = [prometheus.remote_write.mihari.receiver]
+}