yume/yumechi-no-kuni
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 8

Fix usability issues for queue and inbox error metrics #26

Merged
yume merged 1 commit from develop into master 2024-11-18 07:50:24 -06:00
Conversation 0 Commits 1 Files changed 3 +9 -2
3 changed files with 9 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
packages/backend/src/core/QueueModule.ts
View file

@ -18,7 +18,7 @@ import {
SystemWebhookDeliverJobData, SystemWebhookDeliverJobData,
} from '../queue/types.js'; } from '../queue/types.js';
import type { Provider } from '@nestjs/common'; import type { Provider } from '@nestjs/common';
import { mActiveJobs, mDelayedJobs, mJobBlockedCounter, mWaitingJobs } from '@/queue/metrics.js'; import { mActiveJobs, mDelayedJobs, mFailedJobs, mJobBlockedCounter, mWaitingJobs } from '@/queue/metrics.js';
export type SystemQueue = Bull.Queue<Record<string, unknown>>; export type SystemQueue = Bull.Queue<Record<string, unknown>>;
export type EndedPollNotificationQueue = Bull.Queue<EndedPollNotificationJobData>; export type EndedPollNotificationQueue = Bull.Queue<EndedPollNotificationJobData>;
@ -36,6 +36,7 @@ function withMetrics<T>(queue: Bull.Queue<T>): Bull.Queue<T> {
mActiveJobs?.set({ queue: queue.name }, await queue.getActiveCount()); mActiveJobs?.set({ queue: queue.name }, await queue.getActiveCount());
mDelayedJobs?.set({ queue: queue.name }, await queue.getDelayedCount()); mDelayedJobs?.set({ queue: queue.name }, await queue.getDelayedCount());
mWaitingJobs?.set({ queue: queue.name }, await queue.getWaitingCount()); mWaitingJobs?.set({ queue: queue.name }, await queue.getWaitingCount());
mFailedJobs?.set({ queue: queue.name }, await queue.getFailedCount());
}, 2000); }, 2000);
queue.on('waiting', () => { queue.on('waiting', () => {

6
packages/backend/src/queue/metrics.ts
View file

@ -24,6 +24,12 @@ export const mWaitingJobs = metricGauge({
labelNames: ['queue'], labelNames: ['queue'],
}); });
export const mFailedJobs = metricGauge({
name: 'misskey_queue_failed_jobs',
help: 'Total number of failed jobs',
labelNames: ['queue'],
});
export const mStalledWorkerCounter = metricCounter({ export const mStalledWorkerCounter = metricCounter({
name: 'misskey_queue_stalled_workers_total', name: 'misskey_queue_stalled_workers_total',
help: 'Total number of stalled workers', help: 'Total number of stalled workers',

2
packages/backend/src/queue/processors/InboxProcessorService.ts
View file

@ -152,9 +152,9 @@ export class InboxProcessorService implements OnApplicationShutdown {
// 対象が4xxならスキップ // 対象が4xxならスキップ
if (err instanceof StatusError) { if (err instanceof StatusError) {
if (!err.isRetryable) { if (!err.isRetryable) {
incCounter(mIncomingApReject, { reason: 'actor_key_unresolvable' });
throw new Bull.UnrecoverableError(`skip: Ignored deleted actors on both ends ${activity.actor} - ${err.statusCode}`); throw new Bull.UnrecoverableError(`skip: Ignored deleted actors on both ends ${activity.actor} - ${err.statusCode}`);
} }
incCounter(mIncomingApReject, { reason: 'actor_key_unresolvable' });
throw new Error(`Error in actor ${activity.actor} - ${err.statusCode}`); throw new Error(`Error in actor ${activity.actor} - ${err.statusCode}`);
} }
} }