Basic NodeJS instrumentation

Signed-off-by: eternal-flame-AD <yume@yumechi.jp>
This commit is contained in:
ゆめ 2024-11-26 23:55:40 -06:00
parent 4ba3add341
commit b52ca26cb6
No known key found for this signature in database
11 changed files with 214 additions and 14 deletions

View file

@ -147,6 +147,13 @@ redis:
id: 'aidx'
# ┌──────────┐
#───┘ Metrics └──────────────────────────────────────────
#prometheusMetrics:
# enable: false
# scrapeToken: '' # Set non-empty to require a bearer token for scraping
# ┌────────────────┐
#───┘ Error tracking └──────────────────────────────────────────

View file

@ -229,6 +229,13 @@ redis:
id: 'aidx'
# ┌──────────┐
#───┘ Metrics └──────────────────────────────────────────
#prometheusMetrics:
# enable: false
# scrapeToken: '' # Set non-empty to require a bearer token for scraping
# ┌────────────────┐
#───┘ Error tracking └──────────────────────────────────────────

View file

@ -134,8 +134,8 @@
"json5": "2.2.3",
"jsonld": "8.3.2",
"jsrsasign": "11.1.0",
"meilisearch": "0.45.0",
"juice": "11.0.0",
"meilisearch": "0.45.0",
"mfm-js": "0.24.0",
"microformats-parser": "2.0.2",
"mime-types": "2.1.35",
@ -156,6 +156,7 @@
"pg": "8.13.1",
"pkce-challenge": "4.1.0",
"probe-image-size": "7.2.3",
"prom-client": "^15.1.3",
"promise-limit": "2.7.0",
"pug": "3.0.3",
"punycode": "2.3.1",

View file

@ -4,6 +4,7 @@
*/
import { NestFactory } from '@nestjs/core';
import * as prom from 'prom-client';
import { ChartManagementService } from '@/core/chart/ChartManagementService.js';
import { QueueProcessorService } from '@/queue/QueueProcessorService.js';
import { NestLogger } from '@/NestLogger.js';
@ -12,8 +13,9 @@ import { QueueStatsService } from '@/daemons/QueueStatsService.js';
import { ServerStatsService } from '@/daemons/ServerStatsService.js';
import { ServerService } from '@/server/ServerService.js';
import { MainModule } from '@/MainModule.js';
import { MetricsService } from '@/server/api/MetricsService.js';
export async function server() {
export async function server(workerRegistry?: prom.AggregatorRegistry<prom.PrometheusContentType>) {
const app = await NestFactory.createApplicationContext(MainModule, {
logger: new NestLogger(),
});
@ -22,6 +24,9 @@ export async function server() {
await serverService.launch();
if (process.env.NODE_ENV !== 'test') {
if (workerRegistry) {
app.get(MetricsService).setWorkerRegistry(workerRegistry);
}
app.get(ChartManagementService).start();
app.get(QueueStatsService).start();
app.get(ServerStatsService).start();

View file

@ -8,6 +8,7 @@
*/
import cluster from 'node:cluster';
import * as prom from 'prom-client';
import { EventEmitter } from 'node:events';
import chalk from 'chalk';
import Xev from 'xev';
@ -17,6 +18,15 @@ import { masterMain } from './master.js';
import { workerMain } from './worker.js';
import { readyRef } from './ready.js';
const workerRegistry = new prom.AggregatorRegistry<prom.PrometheusContentType>();
prom.collectDefaultMetrics({
labels: {
cluster_type: `${cluster.isPrimary ? 'master' : 'worker'}`,
worker_id: cluster.worker?.id.toString() || 'none'
}
});
import 'reflect-metadata';
process.title = `Misskey (${cluster.isPrimary ? 'master' : 'worker'})`;
@ -69,7 +79,7 @@ process.on('exit', code => {
//#endregion
if (cluster.isPrimary || envOption.disableClustering) {
await masterMain();
await masterMain(workerRegistry);
if (cluster.isPrimary) {
ev.mount();

View file

@ -7,6 +7,7 @@ import * as fs from 'node:fs';
import { fileURLToPath } from 'node:url';
import { dirname } from 'node:path';
import * as os from 'node:os';
import * as prom from 'prom-client';
import cluster from 'node:cluster';
import chalk from 'chalk';
import chalkTemplate from 'chalk-template';
@ -18,6 +19,7 @@ import type { Config } from '@/config.js';
import { showMachineInfo } from '@/misc/show-machine-info.js';
import { envOption } from '@/env.js';
import { jobQueue, server } from './common.js';
import { metricGauge } from '@/server/api/MetricsService.js';
const _filename = fileURLToPath(import.meta.url);
const _dirname = dirname(_filename);
@ -27,7 +29,25 @@ const meta = JSON.parse(fs.readFileSync(`${_dirname}/../../../../built/meta.json
const logger = new Logger('core', 'cyan');
const bootLogger = logger.createSubLogger('boot', 'magenta');
const themeColor = chalk.hex('#f7c3de');
const themeColor = chalk.hex('#86b300');
const mBuildInfo = metricGauge({
name: 'misskey_build_info',
help: 'Misskey build information',
labelNames: ['gitCommit', 'gitDescribe', 'node_version']
});
mBuildInfo?.set({
gitCommit: meta.gitCommit || 'unknown',
gitDescribe: meta.gitDescribe || 'unknown',
node_version: process.version
}, 1);
const mStartupTime = metricGauge({
name: 'misskey_startup_time',
help: 'Misskey startup time',
labelNames: ['pid']
});
function greet() {
if (!envOption.quiet) {
@ -54,7 +74,7 @@ function greet() {
/**
* Init master process
*/
export async function masterMain() {
export async function masterMain(workerRegistry?: prom.AggregatorRegistry<prom.PrometheusContentType>) {
let config!: Config;
// initialize app
@ -64,6 +84,7 @@ export async function masterMain() {
await showMachineInfo(bootLogger);
showNodejsVersion();
config = loadConfigBoot();
//await connectDb();
if (config.pidFile) fs.writeFileSync(config.pidFile, process.pid.toString());
} catch (e) {
@ -91,13 +112,15 @@ export async function masterMain() {
});
}
mStartupTime?.set({ pid: process.pid }, Date.now());
if (envOption.disableClustering) {
if (envOption.onlyServer) {
await server();
await server(workerRegistry);
} else if (envOption.onlyQueue) {
await jobQueue();
} else {
await server();
await server(workerRegistry);
await jobQueue();
}
} else {
@ -106,7 +129,7 @@ export async function masterMain() {
} else if (envOption.onlyQueue) {
// nop
} else {
await server();
await server(workerRegistry);
}
await spawnWorkers(config.clusterLimit);

View file

@ -59,6 +59,7 @@ type Source = {
index: string;
scope?: 'local' | 'global' | string[];
};
prometheusMetrics?: { enable: boolean, scrapeToken?: string };
sentryForBackend?: { options: Partial<Sentry.NodeOptions>; enableNodeProfiling: boolean; };
sentryForFrontend?: { options: Partial<Sentry.NodeOptions> };
@ -184,6 +185,9 @@ export type Config = {
redisForJobQueue: RedisOptions & RedisOptionsSource;
redisForTimelines: RedisOptions & RedisOptionsSource;
redisForReactions: RedisOptions & RedisOptionsSource;
prometheusMetrics : { enable: boolean, scrapeToken?: string } | undefined;
sentryForBackend: { options: Partial<Sentry.NodeOptions>; enableNodeProfiling: boolean; } | undefined;
sentryForFrontend: { options: Partial<Sentry.NodeOptions> } | undefined;
perChannelMaxNoteCacheCount: number;
@ -272,6 +276,7 @@ export function loadConfig(): Config {
redisForJobQueue: config.redisForJobQueue ? convertRedisOptions(config.redisForJobQueue, host) : redis,
redisForTimelines: config.redisForTimelines ? convertRedisOptions(config.redisForTimelines, host) : redis,
redisForReactions: config.redisForReactions ? convertRedisOptions(config.redisForReactions, host) : redis,
prometheusMetrics: config.prometheusMetrics,
sentryForBackend: config.sentryForBackend,
sentryForFrontend: config.sentryForFrontend,
id: config.id,

View file

@ -47,6 +47,7 @@ import { RoleTimelineChannelService } from './api/stream/channels/role-timeline.
import { ReversiChannelService } from './api/stream/channels/reversi.js';
import { ReversiGameChannelService } from './api/stream/channels/reversi-game.js';
import { SigninWithPasskeyApiService } from './api/SigninWithPasskeyApiService.js';
import { MetricsService } from './api/MetricsService.js';
@Module({
imports: [
@ -94,6 +95,7 @@ import { SigninWithPasskeyApiService } from './api/SigninWithPasskeyApiService.j
UserListChannelService,
OpenApiServerService,
OAuth2ProviderService,
MetricsService,
],
exports: [
ServerService,

View file

@ -32,6 +32,7 @@ import { ClientServerService } from './web/ClientServerService.js';
import { OpenApiServerService } from './api/openapi/OpenApiServerService.js';
import { OAuth2ProviderService } from './oauth/OAuth2ProviderService.js';
import { makeHstsHook } from './hsts.js';
import { MetricsService } from './api/MetricsService.js';
const _dirname = fileURLToPath(new URL('.', import.meta.url));
@ -69,6 +70,7 @@ export class ServerService implements OnApplicationShutdown {
private globalEventService: GlobalEventService,
private loggerService: LoggerService,
private oauth2ProviderService: OAuth2ProviderService,
private metricsService: MetricsService,
) {
this.logger = this.loggerService.getLogger('server', 'gray');
}
@ -226,6 +228,8 @@ export class ServerService implements OnApplicationShutdown {
fastify.register(this.clientServerService.createServer);
fastify.register(this.metricsService.createServer);
this.streamingApiServerService.attach(fastify.server);
fastify.server.on('error', err => {

View file

@ -0,0 +1,112 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project and yumechi
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { Inject, Injectable } from "@nestjs/common";
import * as prom from 'prom-client';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import { bindThis } from "@/decorators.js";
import type { FastifyInstance, FastifyPluginOptions } from "fastify";
export function metricGauge<K extends string>(conf: prom.GaugeConfiguration<K>) : prom.Gauge<K> | null {
if (!process.env.RUN_MODE) {
return null;
}
return new prom.Gauge(conf);
}
export function metricCounter<K extends string>(conf: prom.CounterConfiguration<K>) : prom.Counter<K> | null {
if (!process.env.RUN_MODE) {
return null;
}
return new prom.Counter(conf);
}
export function metricHistogram<K extends string>(conf: prom.HistogramConfiguration<K>) : prom.Histogram<K> | null {
if (!process.env.RUN_MODE) {
return null;
}
return new prom.Histogram(conf);
}
@Injectable()
export class MetricsService {
private workerRegistry: prom.AggregatorRegistry<prom.PrometheusContentType> | null = null;
constructor(
@Inject(DI.config)
private config: Config,
) {}
@bindThis
public setWorkerRegistry(workerRegistry: prom.AggregatorRegistry<prom.PrometheusContentType>) {
this.workerRegistry = workerRegistry;
}
@bindThis
public createServer(fastify: FastifyInstance, options: FastifyPluginOptions, done: (err?: Error) => void) {
if (this.config.prometheusMetrics?.enable) {
const token = this.config.prometheusMetrics.scrapeToken;
fastify.get('/metrics', async (request, reply) => {
if (token) {
const bearer = request.headers.authorization;
if (!bearer) {
reply.code(401);
return;
}
const [type, t] = bearer.split(' ');
if (type !== 'Bearer' || t !== token) {
reply.code(403);
return;
}
}
try {
reply.header('Content-Type', prom.register.contentType);
reply.send(await prom.register.metrics());
} catch (err) {
reply.code(500);
}
});
fastify.get('/metrics/cluster', async (request, reply) => {
if (token) {
const bearer = request.headers.authorization;
if (!bearer) {
reply.code(401);
return;
}
const [type, t] = bearer.split(' ');
if (type !== 'Bearer' || t !== token) {
reply.code(403);
return;
}
}
if (!this.workerRegistry) {
reply.code(404);
return;
}
try {
reply.header('Content-Type', this.workerRegistry.contentType);
reply.send(await this.workerRegistry.clusterMetrics());
} catch (err) {
reply.code(500);
}
});
}
done();
}
}

View file

@ -142,7 +142,7 @@ importers:
version: 10.4.7(@nestjs/common@10.4.7(reflect-metadata@0.2.2)(rxjs@7.8.1))(@nestjs/platform-express@10.4.7)(encoding@0.1.13)(reflect-metadata@0.2.2)(rxjs@7.8.1)
'@nestjs/testing':
specifier: 10.4.7
version: 10.4.7(@nestjs/common@10.4.7(reflect-metadata@0.2.2)(rxjs@7.8.1))(@nestjs/core@10.4.7)(@nestjs/platform-express@10.4.7)
version: 10.4.7(@nestjs/common@10.4.7(reflect-metadata@0.2.2)(rxjs@7.8.1))(@nestjs/core@10.4.7(@nestjs/common@10.4.7(reflect-metadata@0.2.2)(rxjs@7.8.1))(@nestjs/platform-express@10.4.7)(encoding@0.1.13)(reflect-metadata@0.2.2)(rxjs@7.8.1))(@nestjs/platform-express@10.4.7(@nestjs/common@10.4.7(reflect-metadata@0.2.2)(rxjs@7.8.1))(@nestjs/core@10.4.7))
'@peertube/http-signature':
specifier: 1.7.0
version: 1.7.0
@ -350,6 +350,9 @@ importers:
probe-image-size:
specifier: 7.2.3
version: 7.2.3
prom-client:
specifier: ^15.1.3
version: 15.1.3
promise-limit:
specifier: 2.7.0
version: 2.7.0
@ -1172,7 +1175,7 @@ importers:
version: 7.17.0(eslint@9.14.0)(typescript@5.6.3)
'@vitest/coverage-v8':
specifier: 1.6.0
version: 1.6.0(vitest@1.6.0(@types/node@22.9.0)(happy-dom@10.0.3)(jsdom@24.1.1(bufferutil@4.0.7)(utf-8-validate@6.0.3))(sass@1.79.4)(terser@5.36.0))
version: 1.6.0(vitest@1.6.0(@types/node@22.9.0)(happy-dom@10.0.3)(jsdom@24.1.1)(sass@1.79.4)(terser@5.36.0))
'@vue/runtime-core':
specifier: 3.5.12
version: 3.5.12
@ -5192,6 +5195,9 @@ packages:
resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==}
engines: {node: '>=8'}
bintrees@1.0.2:
resolution: {integrity: sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==}
blob-util@2.0.2:
resolution: {integrity: sha512-T7JQa+zsXXEa6/8ZhHcQEW1UFfVM49Ts65uBkFL6fz2QmrElqmbajIDJvuA0tEhRe5eIjpV9ZF+0RfZR9voJFQ==}
@ -9009,6 +9015,10 @@ packages:
resolution: {integrity: sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==}
engines: {node: '>=0.4.0'}
prom-client@15.1.3:
resolution: {integrity: sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==}
engines: {node: ^16 || ^18 || >=20}
promise-limit@2.7.0:
resolution: {integrity: sha512-7nJ6v5lnJsXwGprnGXga4wx6d1POjvi5Qmf1ivTRxTjH4Z/9Czja/UCMLVmB9N93GeWOU93XaFaEt6jbuoagNw==}
@ -9978,6 +9988,9 @@ packages:
resolution: {integrity: sha512-+HRtZ40Vc+6YfCDWCeAsixwxJgMbPY4HHuTgzPYH3JXvqHWUlsCfy+ylXlAKhFNcuLp4xVeWeFBUhDk+7KYUvQ==}
engines: {node: '>=14.16'}
tdigest@0.1.2:
resolution: {integrity: sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==}
terser@5.36.0:
resolution: {integrity: sha512-IYV9eNMuFAV4THUspIRXkLakHnV6XO7FEdtKjf/mDyrnqUg9LnlOn6/RwRvM9SZjR4GUq8Nk8zj67FzVARr74w==}
engines: {node: '>=10'}
@ -12721,7 +12734,7 @@ snapshots:
transitivePeerDependencies:
- supports-color
'@nestjs/testing@10.4.7(@nestjs/common@10.4.7(reflect-metadata@0.2.2)(rxjs@7.8.1))(@nestjs/core@10.4.7)(@nestjs/platform-express@10.4.7)':
'@nestjs/testing@10.4.7(@nestjs/common@10.4.7(reflect-metadata@0.2.2)(rxjs@7.8.1))(@nestjs/core@10.4.7(@nestjs/common@10.4.7(reflect-metadata@0.2.2)(rxjs@7.8.1))(@nestjs/platform-express@10.4.7)(encoding@0.1.13)(reflect-metadata@0.2.2)(rxjs@7.8.1))(@nestjs/platform-express@10.4.7(@nestjs/common@10.4.7(reflect-metadata@0.2.2)(rxjs@7.8.1))(@nestjs/core@10.4.7))':
dependencies:
'@nestjs/common': 10.4.7(reflect-metadata@0.2.2)(rxjs@7.8.1)
'@nestjs/core': 10.4.7(@nestjs/common@10.4.7(reflect-metadata@0.2.2)(rxjs@7.8.1))(@nestjs/platform-express@10.4.7)(encoding@0.1.13)(reflect-metadata@0.2.2)(rxjs@7.8.1)
@ -14920,7 +14933,7 @@ snapshots:
transitivePeerDependencies:
- supports-color
'@vitest/coverage-v8@1.6.0(vitest@1.6.0(@types/node@22.9.0)(happy-dom@10.0.3)(jsdom@24.1.1(bufferutil@4.0.7)(utf-8-validate@6.0.3))(sass@1.79.4)(terser@5.36.0))':
'@vitest/coverage-v8@1.6.0(vitest@1.6.0(@types/node@22.9.0)(happy-dom@10.0.3)(jsdom@24.1.1)(sass@1.79.4)(terser@5.36.0))':
dependencies:
'@ampproject/remapping': 2.3.0
'@bcoe/v8-coverage': 0.2.3
@ -14935,7 +14948,7 @@ snapshots:
std-env: 3.8.0
strip-literal: 2.1.0
test-exclude: 6.0.0
vitest: 1.6.0(@types/node@22.9.0)(happy-dom@10.0.3)(jsdom@24.1.1(bufferutil@4.0.7)(utf-8-validate@6.0.3))(sass@1.79.4)(terser@5.36.0)
vitest: 1.6.0(@types/node@22.9.0)(happy-dom@10.0.3)(jsdom@24.1.1)(sass@1.79.4)(terser@5.36.0)
transitivePeerDependencies:
- supports-color
@ -15519,6 +15532,8 @@ snapshots:
binary-extensions@2.3.0: {}
bintrees@1.0.2: {}
blob-util@2.0.2: {}
bluebird@3.7.2: {}
@ -20128,6 +20143,11 @@ snapshots:
progress@2.0.3:
optional: true
prom-client@15.1.3:
dependencies:
'@opentelemetry/api': 1.9.0
tdigest: 0.1.2
promise-limit@2.7.0: {}
promise-polyfill@8.3.0: {}
@ -21250,6 +21270,10 @@ snapshots:
dependencies:
execa: 6.1.0
tdigest@0.1.2:
dependencies:
bintrees: 1.0.2
terser@5.36.0:
dependencies:
'@jridgewell/source-map': 0.3.6
@ -21795,7 +21819,7 @@ snapshots:
- supports-color
- terser
vitest@1.6.0(@types/node@22.9.0)(happy-dom@10.0.3)(jsdom@24.1.1(bufferutil@4.0.7)(utf-8-validate@6.0.3))(sass@1.79.4)(terser@5.36.0):
vitest@1.6.0(@types/node@22.9.0)(happy-dom@10.0.3)(jsdom@24.1.1)(sass@1.79.4)(terser@5.36.0):
dependencies:
'@vitest/expect': 1.6.0
'@vitest/runner': 1.6.0