From b52ca26cb68ae899c07a9bc01966ddd3a468d96b Mon Sep 17 00:00:00 2001 From: eternal-flame-AD Date: Tue, 26 Nov 2024 23:55:40 -0600 Subject: [PATCH] Basic NodeJS instrumentation Signed-off-by: eternal-flame-AD --- .config/docker_example.yml | 7 ++ .config/example.yml | 7 ++ packages/backend/package.json | 3 +- packages/backend/src/boot/common.ts | 7 +- packages/backend/src/boot/entry.ts | 12 +- packages/backend/src/boot/master.ts | 33 +++++- packages/backend/src/config.ts | 5 + packages/backend/src/server/ServerModule.ts | 2 + packages/backend/src/server/ServerService.ts | 4 + .../backend/src/server/api/MetricsService.ts | 112 ++++++++++++++++++ pnpm-lock.yaml | 36 +++++- 11 files changed, 214 insertions(+), 14 deletions(-) create mode 100644 packages/backend/src/server/api/MetricsService.ts diff --git a/.config/docker_example.yml b/.config/docker_example.yml index 5c20894cd..bad7eb253 100644 --- a/.config/docker_example.yml +++ b/.config/docker_example.yml @@ -147,6 +147,13 @@ redis: id: 'aidx' +# ┌──────────┐ +#───┘ Metrics └────────────────────────────────────────── + +#prometheusMetrics: +# enable: false +# scrapeToken: '' # Set non-empty to require a bearer token for scraping + # ┌────────────────┐ #───┘ Error tracking └────────────────────────────────────────── diff --git a/.config/example.yml b/.config/example.yml index e91f96c86..282a52331 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -229,6 +229,13 @@ redis: id: 'aidx' +# ┌──────────┐ +#───┘ Metrics └────────────────────────────────────────── + +#prometheusMetrics: +# enable: false +# scrapeToken: '' # Set non-empty to require a bearer token for scraping + # ┌────────────────┐ #───┘ Error tracking └────────────────────────────────────────── diff --git a/packages/backend/package.json b/packages/backend/package.json index f56a737ee..7fbcdebc7 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -134,8 +134,8 @@ "json5": "2.2.3", "jsonld": "8.3.2", "jsrsasign": "11.1.0", - "meilisearch": "0.45.0", "juice": "11.0.0", + "meilisearch": "0.45.0", "mfm-js": "0.24.0", "microformats-parser": "2.0.2", "mime-types": "2.1.35", @@ -156,6 +156,7 @@ "pg": "8.13.1", "pkce-challenge": "4.1.0", "probe-image-size": "7.2.3", + "prom-client": "^15.1.3", "promise-limit": "2.7.0", "pug": "3.0.3", "punycode": "2.3.1", diff --git a/packages/backend/src/boot/common.ts b/packages/backend/src/boot/common.ts index 268c07582..420701fae 100644 --- a/packages/backend/src/boot/common.ts +++ b/packages/backend/src/boot/common.ts @@ -4,6 +4,7 @@ */ import { NestFactory } from '@nestjs/core'; +import * as prom from 'prom-client'; import { ChartManagementService } from '@/core/chart/ChartManagementService.js'; import { QueueProcessorService } from '@/queue/QueueProcessorService.js'; import { NestLogger } from '@/NestLogger.js'; @@ -12,8 +13,9 @@ import { QueueStatsService } from '@/daemons/QueueStatsService.js'; import { ServerStatsService } from '@/daemons/ServerStatsService.js'; import { ServerService } from '@/server/ServerService.js'; import { MainModule } from '@/MainModule.js'; +import { MetricsService } from '@/server/api/MetricsService.js'; -export async function server() { +export async function server(workerRegistry?: prom.AggregatorRegistry) { const app = await NestFactory.createApplicationContext(MainModule, { logger: new NestLogger(), }); @@ -22,6 +24,9 @@ export async function server() { await serverService.launch(); if (process.env.NODE_ENV !== 'test') { + if (workerRegistry) { + app.get(MetricsService).setWorkerRegistry(workerRegistry); + } app.get(ChartManagementService).start(); app.get(QueueStatsService).start(); app.get(ServerStatsService).start(); diff --git a/packages/backend/src/boot/entry.ts b/packages/backend/src/boot/entry.ts index 25375c301..67570eb9c 100644 --- a/packages/backend/src/boot/entry.ts +++ b/packages/backend/src/boot/entry.ts @@ -8,6 +8,7 @@ */ import cluster from 'node:cluster'; +import * as prom from 'prom-client'; import { EventEmitter } from 'node:events'; import chalk from 'chalk'; import Xev from 'xev'; @@ -17,6 +18,15 @@ import { masterMain } from './master.js'; import { workerMain } from './worker.js'; import { readyRef } from './ready.js'; +const workerRegistry = new prom.AggregatorRegistry(); + +prom.collectDefaultMetrics({ + labels: { + cluster_type: `${cluster.isPrimary ? 'master' : 'worker'}`, + worker_id: cluster.worker?.id.toString() || 'none' + } +}); + import 'reflect-metadata'; process.title = `Misskey (${cluster.isPrimary ? 'master' : 'worker'})`; @@ -69,7 +79,7 @@ process.on('exit', code => { //#endregion if (cluster.isPrimary || envOption.disableClustering) { - await masterMain(); + await masterMain(workerRegistry); if (cluster.isPrimary) { ev.mount(); diff --git a/packages/backend/src/boot/master.ts b/packages/backend/src/boot/master.ts index 41fa5aba4..ddda2afdf 100644 --- a/packages/backend/src/boot/master.ts +++ b/packages/backend/src/boot/master.ts @@ -7,6 +7,7 @@ import * as fs from 'node:fs'; import { fileURLToPath } from 'node:url'; import { dirname } from 'node:path'; import * as os from 'node:os'; +import * as prom from 'prom-client'; import cluster from 'node:cluster'; import chalk from 'chalk'; import chalkTemplate from 'chalk-template'; @@ -18,6 +19,7 @@ import type { Config } from '@/config.js'; import { showMachineInfo } from '@/misc/show-machine-info.js'; import { envOption } from '@/env.js'; import { jobQueue, server } from './common.js'; +import { metricGauge } from '@/server/api/MetricsService.js'; const _filename = fileURLToPath(import.meta.url); const _dirname = dirname(_filename); @@ -27,7 +29,25 @@ const meta = JSON.parse(fs.readFileSync(`${_dirname}/../../../../built/meta.json const logger = new Logger('core', 'cyan'); const bootLogger = logger.createSubLogger('boot', 'magenta'); -const themeColor = chalk.hex('#f7c3de'); +const themeColor = chalk.hex('#86b300'); + +const mBuildInfo = metricGauge({ + name: 'misskey_build_info', + help: 'Misskey build information', + labelNames: ['gitCommit', 'gitDescribe', 'node_version'] +}); + +mBuildInfo?.set({ + gitCommit: meta.gitCommit || 'unknown', + gitDescribe: meta.gitDescribe || 'unknown', + node_version: process.version +}, 1); + +const mStartupTime = metricGauge({ + name: 'misskey_startup_time', + help: 'Misskey startup time', + labelNames: ['pid'] +}); function greet() { if (!envOption.quiet) { @@ -54,7 +74,7 @@ function greet() { /** * Init master process */ -export async function masterMain() { +export async function masterMain(workerRegistry?: prom.AggregatorRegistry) { let config!: Config; // initialize app @@ -64,6 +84,7 @@ export async function masterMain() { await showMachineInfo(bootLogger); showNodejsVersion(); config = loadConfigBoot(); + //await connectDb(); if (config.pidFile) fs.writeFileSync(config.pidFile, process.pid.toString()); } catch (e) { @@ -91,13 +112,15 @@ export async function masterMain() { }); } + mStartupTime?.set({ pid: process.pid }, Date.now()); + if (envOption.disableClustering) { if (envOption.onlyServer) { - await server(); + await server(workerRegistry); } else if (envOption.onlyQueue) { await jobQueue(); } else { - await server(); + await server(workerRegistry); await jobQueue(); } } else { @@ -106,7 +129,7 @@ export async function masterMain() { } else if (envOption.onlyQueue) { // nop } else { - await server(); + await server(workerRegistry); } await spawnWorkers(config.clusterLimit); diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index ee91bd094..1ea1269c0 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -59,6 +59,7 @@ type Source = { index: string; scope?: 'local' | 'global' | string[]; }; + prometheusMetrics?: { enable: boolean, scrapeToken?: string }; sentryForBackend?: { options: Partial; enableNodeProfiling: boolean; }; sentryForFrontend?: { options: Partial }; @@ -184,6 +185,9 @@ export type Config = { redisForJobQueue: RedisOptions & RedisOptionsSource; redisForTimelines: RedisOptions & RedisOptionsSource; redisForReactions: RedisOptions & RedisOptionsSource; + + prometheusMetrics : { enable: boolean, scrapeToken?: string } | undefined; + sentryForBackend: { options: Partial; enableNodeProfiling: boolean; } | undefined; sentryForFrontend: { options: Partial } | undefined; perChannelMaxNoteCacheCount: number; @@ -272,6 +276,7 @@ export function loadConfig(): Config { redisForJobQueue: config.redisForJobQueue ? convertRedisOptions(config.redisForJobQueue, host) : redis, redisForTimelines: config.redisForTimelines ? convertRedisOptions(config.redisForTimelines, host) : redis, redisForReactions: config.redisForReactions ? convertRedisOptions(config.redisForReactions, host) : redis, + prometheusMetrics: config.prometheusMetrics, sentryForBackend: config.sentryForBackend, sentryForFrontend: config.sentryForFrontend, id: config.id, diff --git a/packages/backend/src/server/ServerModule.ts b/packages/backend/src/server/ServerModule.ts index 3ab0b815f..df1d0c0d5 100644 --- a/packages/backend/src/server/ServerModule.ts +++ b/packages/backend/src/server/ServerModule.ts @@ -47,6 +47,7 @@ import { RoleTimelineChannelService } from './api/stream/channels/role-timeline. import { ReversiChannelService } from './api/stream/channels/reversi.js'; import { ReversiGameChannelService } from './api/stream/channels/reversi-game.js'; import { SigninWithPasskeyApiService } from './api/SigninWithPasskeyApiService.js'; +import { MetricsService } from './api/MetricsService.js'; @Module({ imports: [ @@ -94,6 +95,7 @@ import { SigninWithPasskeyApiService } from './api/SigninWithPasskeyApiService.j UserListChannelService, OpenApiServerService, OAuth2ProviderService, + MetricsService, ], exports: [ ServerService, diff --git a/packages/backend/src/server/ServerService.ts b/packages/backend/src/server/ServerService.ts index a733adbc4..62a257676 100644 --- a/packages/backend/src/server/ServerService.ts +++ b/packages/backend/src/server/ServerService.ts @@ -32,6 +32,7 @@ import { ClientServerService } from './web/ClientServerService.js'; import { OpenApiServerService } from './api/openapi/OpenApiServerService.js'; import { OAuth2ProviderService } from './oauth/OAuth2ProviderService.js'; import { makeHstsHook } from './hsts.js'; +import { MetricsService } from './api/MetricsService.js'; const _dirname = fileURLToPath(new URL('.', import.meta.url)); @@ -69,6 +70,7 @@ export class ServerService implements OnApplicationShutdown { private globalEventService: GlobalEventService, private loggerService: LoggerService, private oauth2ProviderService: OAuth2ProviderService, + private metricsService: MetricsService, ) { this.logger = this.loggerService.getLogger('server', 'gray'); } @@ -226,6 +228,8 @@ export class ServerService implements OnApplicationShutdown { fastify.register(this.clientServerService.createServer); + fastify.register(this.metricsService.createServer); + this.streamingApiServerService.attach(fastify.server); fastify.server.on('error', err => { diff --git a/packages/backend/src/server/api/MetricsService.ts b/packages/backend/src/server/api/MetricsService.ts new file mode 100644 index 000000000..c2021cef1 --- /dev/null +++ b/packages/backend/src/server/api/MetricsService.ts @@ -0,0 +1,112 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project and yumechi + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Inject, Injectable } from "@nestjs/common"; +import * as prom from 'prom-client'; +import { DI } from '@/di-symbols.js'; +import type { Config } from '@/config.js'; +import { bindThis } from "@/decorators.js"; +import type { FastifyInstance, FastifyPluginOptions } from "fastify"; + +export function metricGauge(conf: prom.GaugeConfiguration) : prom.Gauge | null { + if (!process.env.RUN_MODE) { + return null; + } + + return new prom.Gauge(conf); +} + +export function metricCounter(conf: prom.CounterConfiguration) : prom.Counter | null { + if (!process.env.RUN_MODE) { + return null; + } + + return new prom.Counter(conf); +} + +export function metricHistogram(conf: prom.HistogramConfiguration) : prom.Histogram | null { + if (!process.env.RUN_MODE) { + return null; + } + + return new prom.Histogram(conf); +} + +@Injectable() +export class MetricsService { + private workerRegistry: prom.AggregatorRegistry | null = null; + constructor( + @Inject(DI.config) + private config: Config, + ) {} + + @bindThis + public setWorkerRegistry(workerRegistry: prom.AggregatorRegistry) { + this.workerRegistry = workerRegistry; + } + + @bindThis + public createServer(fastify: FastifyInstance, options: FastifyPluginOptions, done: (err?: Error) => void) { + if (this.config.prometheusMetrics?.enable) { + const token = this.config.prometheusMetrics.scrapeToken; + fastify.get('/metrics', async (request, reply) => { + if (token) { + const bearer = request.headers.authorization; + + if (!bearer) { + reply.code(401); + return; + } + + const [type, t] = bearer.split(' '); + + if (type !== 'Bearer' || t !== token) { + reply.code(403); + return; + } + } + + try { + reply.header('Content-Type', prom.register.contentType); + reply.send(await prom.register.metrics()); + } catch (err) { + reply.code(500); + } + }); + + fastify.get('/metrics/cluster', async (request, reply) => { + if (token) { + const bearer = request.headers.authorization; + + if (!bearer) { + reply.code(401); + return; + } + + const [type, t] = bearer.split(' '); + + if (type !== 'Bearer' || t !== token) { + reply.code(403); + return; + } + } + + if (!this.workerRegistry) { + reply.code(404); + return; + } + + try { + reply.header('Content-Type', this.workerRegistry.contentType); + reply.send(await this.workerRegistry.clusterMetrics()); + } catch (err) { + reply.code(500); + } + }); + } + + done(); + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 451d83efc..448fb8775 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -142,7 +142,7 @@ importers: version: 10.4.7(@nestjs/common@10.4.7(reflect-metadata@0.2.2)(rxjs@7.8.1))(@nestjs/platform-express@10.4.7)(encoding@0.1.13)(reflect-metadata@0.2.2)(rxjs@7.8.1) '@nestjs/testing': specifier: 10.4.7 - version: 10.4.7(@nestjs/common@10.4.7(reflect-metadata@0.2.2)(rxjs@7.8.1))(@nestjs/core@10.4.7)(@nestjs/platform-express@10.4.7) + version: 10.4.7(@nestjs/common@10.4.7(reflect-metadata@0.2.2)(rxjs@7.8.1))(@nestjs/core@10.4.7(@nestjs/common@10.4.7(reflect-metadata@0.2.2)(rxjs@7.8.1))(@nestjs/platform-express@10.4.7)(encoding@0.1.13)(reflect-metadata@0.2.2)(rxjs@7.8.1))(@nestjs/platform-express@10.4.7(@nestjs/common@10.4.7(reflect-metadata@0.2.2)(rxjs@7.8.1))(@nestjs/core@10.4.7)) '@peertube/http-signature': specifier: 1.7.0 version: 1.7.0 @@ -350,6 +350,9 @@ importers: probe-image-size: specifier: 7.2.3 version: 7.2.3 + prom-client: + specifier: ^15.1.3 + version: 15.1.3 promise-limit: specifier: 2.7.0 version: 2.7.0 @@ -1172,7 +1175,7 @@ importers: version: 7.17.0(eslint@9.14.0)(typescript@5.6.3) '@vitest/coverage-v8': specifier: 1.6.0 - version: 1.6.0(vitest@1.6.0(@types/node@22.9.0)(happy-dom@10.0.3)(jsdom@24.1.1(bufferutil@4.0.7)(utf-8-validate@6.0.3))(sass@1.79.4)(terser@5.36.0)) + version: 1.6.0(vitest@1.6.0(@types/node@22.9.0)(happy-dom@10.0.3)(jsdom@24.1.1)(sass@1.79.4)(terser@5.36.0)) '@vue/runtime-core': specifier: 3.5.12 version: 3.5.12 @@ -5192,6 +5195,9 @@ packages: resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==} engines: {node: '>=8'} + bintrees@1.0.2: + resolution: {integrity: sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==} + blob-util@2.0.2: resolution: {integrity: sha512-T7JQa+zsXXEa6/8ZhHcQEW1UFfVM49Ts65uBkFL6fz2QmrElqmbajIDJvuA0tEhRe5eIjpV9ZF+0RfZR9voJFQ==} @@ -9009,6 +9015,10 @@ packages: resolution: {integrity: sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==} engines: {node: '>=0.4.0'} + prom-client@15.1.3: + resolution: {integrity: sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==} + engines: {node: ^16 || ^18 || >=20} + promise-limit@2.7.0: resolution: {integrity: sha512-7nJ6v5lnJsXwGprnGXga4wx6d1POjvi5Qmf1ivTRxTjH4Z/9Czja/UCMLVmB9N93GeWOU93XaFaEt6jbuoagNw==} @@ -9978,6 +9988,9 @@ packages: resolution: {integrity: sha512-+HRtZ40Vc+6YfCDWCeAsixwxJgMbPY4HHuTgzPYH3JXvqHWUlsCfy+ylXlAKhFNcuLp4xVeWeFBUhDk+7KYUvQ==} engines: {node: '>=14.16'} + tdigest@0.1.2: + resolution: {integrity: sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==} + terser@5.36.0: resolution: {integrity: sha512-IYV9eNMuFAV4THUspIRXkLakHnV6XO7FEdtKjf/mDyrnqUg9LnlOn6/RwRvM9SZjR4GUq8Nk8zj67FzVARr74w==} engines: {node: '>=10'} @@ -12721,7 +12734,7 @@ snapshots: transitivePeerDependencies: - supports-color - '@nestjs/testing@10.4.7(@nestjs/common@10.4.7(reflect-metadata@0.2.2)(rxjs@7.8.1))(@nestjs/core@10.4.7)(@nestjs/platform-express@10.4.7)': + '@nestjs/testing@10.4.7(@nestjs/common@10.4.7(reflect-metadata@0.2.2)(rxjs@7.8.1))(@nestjs/core@10.4.7(@nestjs/common@10.4.7(reflect-metadata@0.2.2)(rxjs@7.8.1))(@nestjs/platform-express@10.4.7)(encoding@0.1.13)(reflect-metadata@0.2.2)(rxjs@7.8.1))(@nestjs/platform-express@10.4.7(@nestjs/common@10.4.7(reflect-metadata@0.2.2)(rxjs@7.8.1))(@nestjs/core@10.4.7))': dependencies: '@nestjs/common': 10.4.7(reflect-metadata@0.2.2)(rxjs@7.8.1) '@nestjs/core': 10.4.7(@nestjs/common@10.4.7(reflect-metadata@0.2.2)(rxjs@7.8.1))(@nestjs/platform-express@10.4.7)(encoding@0.1.13)(reflect-metadata@0.2.2)(rxjs@7.8.1) @@ -14920,7 +14933,7 @@ snapshots: transitivePeerDependencies: - supports-color - '@vitest/coverage-v8@1.6.0(vitest@1.6.0(@types/node@22.9.0)(happy-dom@10.0.3)(jsdom@24.1.1(bufferutil@4.0.7)(utf-8-validate@6.0.3))(sass@1.79.4)(terser@5.36.0))': + '@vitest/coverage-v8@1.6.0(vitest@1.6.0(@types/node@22.9.0)(happy-dom@10.0.3)(jsdom@24.1.1)(sass@1.79.4)(terser@5.36.0))': dependencies: '@ampproject/remapping': 2.3.0 '@bcoe/v8-coverage': 0.2.3 @@ -14935,7 +14948,7 @@ snapshots: std-env: 3.8.0 strip-literal: 2.1.0 test-exclude: 6.0.0 - vitest: 1.6.0(@types/node@22.9.0)(happy-dom@10.0.3)(jsdom@24.1.1(bufferutil@4.0.7)(utf-8-validate@6.0.3))(sass@1.79.4)(terser@5.36.0) + vitest: 1.6.0(@types/node@22.9.0)(happy-dom@10.0.3)(jsdom@24.1.1)(sass@1.79.4)(terser@5.36.0) transitivePeerDependencies: - supports-color @@ -15519,6 +15532,8 @@ snapshots: binary-extensions@2.3.0: {} + bintrees@1.0.2: {} + blob-util@2.0.2: {} bluebird@3.7.2: {} @@ -20128,6 +20143,11 @@ snapshots: progress@2.0.3: optional: true + prom-client@15.1.3: + dependencies: + '@opentelemetry/api': 1.9.0 + tdigest: 0.1.2 + promise-limit@2.7.0: {} promise-polyfill@8.3.0: {} @@ -21250,6 +21270,10 @@ snapshots: dependencies: execa: 6.1.0 + tdigest@0.1.2: + dependencies: + bintrees: 1.0.2 + terser@5.36.0: dependencies: '@jridgewell/source-map': 0.3.6 @@ -21795,7 +21819,7 @@ snapshots: - supports-color - terser - vitest@1.6.0(@types/node@22.9.0)(happy-dom@10.0.3)(jsdom@24.1.1(bufferutil@4.0.7)(utf-8-validate@6.0.3))(sass@1.79.4)(terser@5.36.0): + vitest@1.6.0(@types/node@22.9.0)(happy-dom@10.0.3)(jsdom@24.1.1)(sass@1.79.4)(terser@5.36.0): dependencies: '@vitest/expect': 1.6.0 '@vitest/runner': 1.6.0